diff --git a/.github/actions/e2e/cleanup/action.yaml b/.github/actions/e2e/cleanup/action.yaml index 7237012066c6..f74292acfe8d 100644 --- a/.github/actions/e2e/cleanup/action.yaml +++ b/.github/actions/e2e/cleanup/action.yaml @@ -17,14 +17,14 @@ inputs: description: "The git commit, tag, or branch to check out. Requires a corresponding Karpenter snapshot release" eksctl_version: description: "Version of eksctl to install" - default: v0.169.0 + default: v0.180.0 private_cluster: description: "Whether the cluster that has to be deleted is private or not. Valid values are 'true' or 'false'" default: 'false' runs: using: "composite" steps: - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: ref: ${{ inputs.git_ref }} - uses: ./.github/actions/e2e/install-eksctl @@ -37,7 +37,7 @@ runs: CLUSTER_NAME: ${{ inputs.cluster_name }} run: | eksctl delete cluster --name "$CLUSTER_NAME" --timeout 60m --wait || true - - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version-file: test/hack/resource/go.mod cache-dependency-path: test/hack/resource/go.sum diff --git a/.github/actions/e2e/install-karpenter/action.yaml b/.github/actions/e2e/install-karpenter/action.yaml index 137f6237bb26..f134dd604a1b 100644 --- a/.github/actions/e2e/install-karpenter/action.yaml +++ b/.github/actions/e2e/install-karpenter/action.yaml @@ -21,7 +21,7 @@ inputs: required: true k8s_version: description: 'Version of Kubernetes to use for the launched cluster' - default: "1.29" + default: "1.30" git_ref: description: "The git commit, tag, or branch to check out. Requires a corresponding Karpenter snapshot release" private_cluster: @@ -30,7 +30,7 @@ inputs: runs: using: "composite" steps: - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: ref: ${{ inputs.git_ref }} - uses: ./.github/actions/e2e/install-helm @@ -44,7 +44,7 @@ runs: kubectl label ns kube-system scrape=enabled --overwrite=true kubectl label ns kube-system pod-security.kubernetes.io/warn=restricted --overwrite=true - name: login to ecr via docker - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: ${{ inputs.ecr_account_id }}.dkr.ecr.${{ inputs.ecr_region }}.amazonaws.com logout: true @@ -55,7 +55,6 @@ runs: ECR_REGION: ${{ inputs.ecr_region }} ACCOUNT_ID: ${{ inputs.account_id }} CLUSTER_NAME: ${{ inputs.cluster_name }} - K8S_VERSION: ${{ inputs.k8s_version }} PRIVATE_CLUSTER: ${{ inputs.private_cluster }} run: | ./test/hack/e2e_scripts/install_karpenter.sh diff --git a/.github/actions/e2e/install-prometheus/action.yaml b/.github/actions/e2e/install-prometheus/action.yaml index f80dd138c8a6..3c4a84cf7122 100644 --- a/.github/actions/e2e/install-prometheus/action.yaml +++ b/.github/actions/e2e/install-prometheus/action.yaml @@ -27,7 +27,7 @@ inputs: runs: using: "composite" steps: - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: ref: ${{ inputs.git_ref }} - uses: ./.github/actions/e2e/install-helm diff --git a/.github/actions/e2e/install-prometheus/values.yaml b/.github/actions/e2e/install-prometheus/values.yaml index 5a6656cf742e..97570b00d9fd 100644 --- a/.github/actions/e2e/install-prometheus/values.yaml +++ b/.github/actions/e2e/install-prometheus/values.yaml @@ -47,10 +47,10 @@ prometheus: resources: requests: cpu: 1 - memory: 5Gi + memory: 15Gi limits: cpu: 1 - memory: 5Gi + memory: 15Gi serviceMonitorSelector: matchLabels: scrape: enabled diff --git a/.github/actions/e2e/run-tests-private-cluster/action.yaml b/.github/actions/e2e/run-tests-private-cluster/action.yaml index 64504b9b6d58..86da1447d471 100644 --- a/.github/actions/e2e/run-tests-private-cluster/action.yaml +++ b/.github/actions/e2e/run-tests-private-cluster/action.yaml @@ -33,7 +33,7 @@ inputs: required: true k8s_version: description: 'Version of Kubernetes to use for the launched cluster' - default: "1.29" + default: "1.30" private_cluster: description: "Whether to create a private cluster which does not add access to the public internet. Valid values are 'true' or 'false'" default: 'false' @@ -53,7 +53,7 @@ runs: using: "composite" steps: - name: login to ecr via docker - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: ${{ inputs.account_id }}.dkr.ecr.${{ inputs.region }}.amazonaws.com logout: true @@ -93,7 +93,7 @@ runs: CLUSTER_VPC_ID: ${{ env.CLUSTER_VPC_ID }} EKS_CLUSTER_SG: ${{ env.EKS_CLUSTER_SG }} CLEANUP: ${{ inputs.cleanup }} - uses: aws-actions/aws-codebuild-run-build@b0a7ca5730725c01b45af8866100667e32e0a9b1 #v1.0.15 + uses: aws-actions/aws-codebuild-run-build@f59b837dd074776bd06619e7e22fb62161eab324 #v1.0.15 with: project-name: E2EPrivateClusterCodeBuildProject-us-east-1 buildspec-override: | @@ -156,4 +156,4 @@ runs: VPC_CB, CLUSTER_VPC_ID, EKS_CLUSTER_SG, - CLEANUP \ No newline at end of file + CLEANUP diff --git a/.github/actions/e2e/setup-cluster/action.yaml b/.github/actions/e2e/setup-cluster/action.yaml index c829662294f0..ff3d656d5d70 100644 --- a/.github/actions/e2e/setup-cluster/action.yaml +++ b/.github/actions/e2e/setup-cluster/action.yaml @@ -27,10 +27,10 @@ inputs: required: true k8s_version: description: 'Version of Kubernetes to use for the launched cluster' - default: "1.29" + default: "1.30" eksctl_version: description: "Version of eksctl to install" - default: v0.169.0 + default: v0.180.0 ip_family: description: "IP Family of the cluster. Valid values are IPv4 or IPv6" default: "IPv4" @@ -50,7 +50,7 @@ inputs: runs: using: "composite" steps: - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: ref: ${{ inputs.git_ref }} - uses: ./.github/actions/e2e/install-eksctl @@ -64,9 +64,6 @@ runs: run: | # Resolve the cloudformation path with fallback CLOUDFORMATION_PATH=website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml - if [ ! -f $CLOUDFORMATION_PATH ]; then - CLOUDFORMATION_PATH=website/content/en/preview/getting-started/getting-started-with-eksctl/cloudformation.yaml - fi # Update the Cloudformation policy to add the permissionBoundary to the NodeRole yq -i ".Resources.KarpenterNodeRole.Properties.PermissionsBoundary = \"arn:aws:iam::$ACCOUNT_ID:policy/GithubActionsPermissionsBoundary\"" $CLOUDFORMATION_PATH @@ -96,9 +93,9 @@ runs: GIT_REF=$(git rev-parse HEAD) fi - # Disable Pod Identity for Karpenter on K8s 1.23. Pod Identity is not supported on K8s 1.23 + # Disable Pod Identity for Private Clusters # https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html#pod-id-considerations - if [[ "$K8S_VERSION" == '1.23' ]] || [[ "$PRIVATE_CLUSTER" == 'true' ]]; then + if [[ "$PRIVATE_CLUSTER" == 'true' ]]; then KARPENTER_IAM=""" - metadata: name: karpenter diff --git a/.github/actions/e2e/slack/notify/action.yaml b/.github/actions/e2e/slack/notify/action.yaml index 3fadcd90954f..9cd0d14046b4 100644 --- a/.github/actions/e2e/slack/notify/action.yaml +++ b/.github/actions/e2e/slack/notify/action.yaml @@ -17,7 +17,7 @@ inputs: runs: using: "composite" steps: - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: ref: ${{ inputs.git_ref }} - id: get-run-name diff --git a/.github/actions/e2e/upgrade-crds/action.yaml b/.github/actions/e2e/upgrade-crds/action.yaml index 23c28d7fb1c5..122d23e25c2f 100644 --- a/.github/actions/e2e/upgrade-crds/action.yaml +++ b/.github/actions/e2e/upgrade-crds/action.yaml @@ -24,7 +24,7 @@ runs: role-to-assume: arn:aws:iam::${{ inputs.account_id }}:role/${{ inputs.role }} aws-region: ${{ inputs.region }} role-duration-seconds: 21600 - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: ref: ${{ inputs.git_ref }} - name: install-karpenter diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml index f0dd71b3b83c..3475f09efc49 100644 --- a/.github/actions/install-deps/action.yaml +++ b/.github/actions/install-deps/action.yaml @@ -3,11 +3,11 @@ description: 'Installs Go Downloads and installs Karpenter Dependencies' inputs: k8sVersion: description: Kubernetes version to use when installing the toolchain - default: "1.29.x" + default: "1.30.x" runs: using: "composite" steps: - - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 id: setup-go with: go-version-file: go.mod diff --git a/.github/workflows/approval-comment.yaml b/.github/workflows/approval-comment.yaml index 34dfcafe9c05..6e59fd3181e7 100644 --- a/.github/workflows/approval-comment.yaml +++ b/.github/workflows/approval-comment.yaml @@ -19,7 +19,7 @@ jobs: mkdir -p /tmp/artifacts { echo "$REVIEW_BODY"; echo "$PULL_REQUEST_NUMBER"; echo "$COMMIT_ID"; } >> /tmp/artifacts/metadata.txt cat /tmp/artifacts/metadata.txt - - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: artifacts path: /tmp/artifacts diff --git a/.github/workflows/ci-test.yaml b/.github/workflows/ci-test.yaml index 58cfdb9b204b..36fa38add4c0 100644 --- a/.github/workflows/ci-test.yaml +++ b/.github/workflows/ci-test.yaml @@ -4,6 +4,7 @@ on: branches: - 'main' - 'release-v*' + - 'staging/*' pull_request: workflow_dispatch: jobs: @@ -11,8 +12,9 @@ jobs: runs-on: ubuntu-latest if: github.repository == 'aws/karpenter-provider-aws' strategy: + fail-fast: false matrix: - k8sVersion: ["1.23.x", "1.24.x", "1.25.x", "1.26.x", "1.27.x", "1.28.x", "1.29.x"] + k8sVersion: ["1.25.x", "1.26.x", "1.27.x", "1.28.x", "1.29.x", "1.30.x"] steps: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - uses: ./.github/actions/install-deps @@ -21,7 +23,7 @@ jobs: - run: K8S_VERSION=${{ matrix.k8sVersion }} make ci-test - name: Send coverage # should only send converage once https://docs.coveralls.io/parallel-builds - if: matrix.k8sVersion == '1.29.x' + if: matrix.k8sVersion == '1.30.x' env: COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: goveralls -coverprofile=coverage.out -service=github diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 55fcdb828096..282c841a77fd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,6 +4,7 @@ on: branches: - 'main' - 'release-v*' + - 'staging/*' pull_request: workflow_dispatch: jobs: diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml index c78fc63f866d..c30885c66caa 100644 --- a/.github/workflows/codeql-analysis.yaml +++ b/.github/workflows/codeql-analysis.yaml @@ -4,6 +4,7 @@ on: branches: - 'main' - 'release-v*' + - 'staging/*' pull_request: schedule: - cron: '0 12 * * *' diff --git a/.github/workflows/dryrun-gen.yaml b/.github/workflows/dryrun-gen.yaml index b18c62b3697e..4501ea4d79cf 100644 --- a/.github/workflows/dryrun-gen.yaml +++ b/.github/workflows/dryrun-gen.yaml @@ -4,6 +4,7 @@ on: branches: - 'main' - 'release-v*' + - 'staging/*' jobs: dryrun-gen: permissions: diff --git a/.github/workflows/e2e-cleanup.yaml b/.github/workflows/e2e-cleanup.yaml index a5844c6331bf..21b0f053abdd 100644 --- a/.github/workflows/e2e-cleanup.yaml +++ b/.github/workflows/e2e-cleanup.yaml @@ -38,4 +38,4 @@ jobs: region: ${{ inputs.region }} cluster_name: ${{ inputs.cluster_name }} git_ref: ${{ inputs.git_ref }} - eksctl_version: v0.169.0 + eksctl_version: v0.180.0 diff --git a/.github/workflows/e2e-matrix-trigger.yaml b/.github/workflows/e2e-matrix-trigger.yaml index 99b31383ba49..e8580e0c5370 100644 --- a/.github/workflows/e2e-matrix-trigger.yaml +++ b/.github/workflows/e2e-matrix-trigger.yaml @@ -6,6 +6,7 @@ on: branches: - 'main' - 'release-v*' + - 'staging/*' workflow_run: workflows: - ApprovalComment diff --git a/.github/workflows/e2e-matrix.yaml b/.github/workflows/e2e-matrix.yaml index 475edab660e4..df0e72782b45 100644 --- a/.github/workflows/e2e-matrix.yaml +++ b/.github/workflows/e2e-matrix.yaml @@ -7,7 +7,7 @@ on: default: "us-east-2" k8s_version: type: string - default: "1.29" + default: "1.30" cleanup: type: boolean required: true @@ -34,14 +34,13 @@ on: k8s_version: type: choice options: - - "1.23" - - "1.24" - "1.25" - "1.26" - "1.27" - "1.28" - "1.29" - default: "1.29" + - "1.30" + default: "1.30" cleanup: type: boolean required: true @@ -60,6 +59,14 @@ jobs: max-parallel: ${{ inputs.parallelism || 100 }} matrix: suite: + - name: IPv6 + region: ${{ inputs.region }} + - name: AMI + region: ${{ inputs.region }} + - name: Scheduling + region: ${{ inputs.region }} + - name: Storage + region: ${{ inputs.region }} - name: Integration region: ${{ inputs.region }} - name: NodeClaim @@ -74,7 +81,7 @@ jobs: region: ${{ inputs.region }} - name: Chaos region: ${{ inputs.region }} - - name: IPv6 + - name: Termination region: ${{ inputs.region }} - name: LocalZone # LAX is the only local zone available in the CI account, therefore only use us-west-2 @@ -95,7 +102,7 @@ jobs: statuses: write # ./.github/actions/commit-status/start uses: ./.github/workflows/e2e-upgrade.yaml with: - from_git_ref: 283e7b2a51ec73903a6d3f9362fc3009b898ef33 + from_git_ref: b3076dca62a81caae2d3c4af4fd378c83a901c48 to_git_ref: ${{ inputs.git_ref }} region: ${{ inputs.region }} k8s_version: ${{ inputs.k8s_version }} diff --git a/.github/workflows/e2e-soak-trigger.yaml b/.github/workflows/e2e-soak-trigger.yaml index 862473a2c2dd..490c069167d0 100644 --- a/.github/workflows/e2e-soak-trigger.yaml +++ b/.github/workflows/e2e-soak-trigger.yaml @@ -17,7 +17,7 @@ jobs: with: role-to-assume: arn:aws:iam::${{ vars.CI_ACCOUNT_ID }}:role/${{ vars.CI_ROLE_NAME }} aws-region: eu-north-1 - - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version-file: test/hack/soak/go.mod cache-dependency-path: test/hack/soak/go.sum diff --git a/.github/workflows/e2e-upgrade.yaml b/.github/workflows/e2e-upgrade.yaml index 0ad5a6e3f3e0..f5751de20aac 100644 --- a/.github/workflows/e2e-upgrade.yaml +++ b/.github/workflows/e2e-upgrade.yaml @@ -16,14 +16,13 @@ on: k8s_version: type: choice options: - - "1.23" - - "1.24" - "1.25" - "1.26" - "1.27" - "1.28" - "1.29" - default: "1.29" + - "1.30" + default: "1.30" cleanup: required: true default: true @@ -40,7 +39,7 @@ on: default: "us-east-2" k8s_version: type: string - default: "1.29" + default: "1.30" cleanup: required: true type: boolean @@ -90,7 +89,7 @@ jobs: region: ${{ inputs.region }} cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} k8s_version: ${{ inputs.k8s_version }} - eksctl_version: v0.169.0 + eksctl_version: v0.180.0 ip_family: IPv4 # Set the value to IPv6 if IPv6 suite, else IPv4 git_ref: ${{ inputs.from_git_ref }} ecr_account_id: ${{ vars.SNAPSHOT_ACCOUNT_ID }} @@ -100,6 +99,14 @@ jobs: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ inputs.to_git_ref }} + - name: upgrade crds + uses: ./.github/actions/e2e/upgrade-crds + with: + account_id: ${{ vars.CI_ACCOUNT_ID }} + role: ${{ vars.CI_ROLE_NAME }} + region: ${{ inputs.region }} + cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} + git_ref: ${{ inputs.to_git_ref }} - name: upgrade eks cluster '${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }}' uses: ./.github/actions/e2e/setup-cluster with: @@ -108,21 +115,13 @@ jobs: region: ${{ inputs.region }} cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} k8s_version: ${{ inputs.k8s_version }} - eksctl_version: v0.169.0 + eksctl_version: v0.180.0 ip_family: IPv4 # Set the value to IPv6 if IPv6 suite, else IPv4 git_ref: ${{ inputs.to_git_ref }} ecr_account_id: ${{ vars.SNAPSHOT_ACCOUNT_ID }} ecr_region: ${{ vars.SNAPSHOT_REGION }} prometheus_workspace_id: ${{ vars.WORKSPACE_ID }} prometheus_region: ${{ vars.PROMETHEUS_REGION }} - - name: upgrade crds - uses: ./.github/actions/e2e/upgrade-crds - with: - account_id: ${{ vars.CI_ACCOUNT_ID }} - role: ${{ vars.CI_ROLE_NAME }} - region: ${{ inputs.region }} - cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} - git_ref: ${{ inputs.to_git_ref }} - name: run the Upgrade test suite run: | aws eks update-kubeconfig --name ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} @@ -152,7 +151,7 @@ jobs: region: ${{ inputs.region }} cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} git_ref: ${{ inputs.to_git_ref }} - eksctl_version: v0.169.0 + eksctl_version: v0.180.0 - if: always() && github.event_name == 'workflow_run' uses: ./.github/actions/commit-status/end with: diff --git a/.github/workflows/e2e-version-compatibility-trigger.yaml b/.github/workflows/e2e-version-compatibility-trigger.yaml index e4b9f59ddab8..caa95f24509e 100644 --- a/.github/workflows/e2e-version-compatibility-trigger.yaml +++ b/.github/workflows/e2e-version-compatibility-trigger.yaml @@ -34,14 +34,15 @@ jobs: strategy: fail-fast: false matrix: - k8s_version: [ "1.23", "1.24", "1.25", "1.26", "1.27", "1.28", "1.29"] + k8s_version: ["1.25", "1.26", "1.27", "1.28", "1.29", "1.30"] uses: ./.github/workflows/e2e-matrix.yaml with: region: ${{ inputs.region || 'eu-west-1' }} + git_ref: ${{ needs.resolve.outputs.GIT_REF }} k8s_version: ${{ matrix.k8s_version }} workflow_trigger: "versionCompatibility" # Default to true unless using a workflow_dispatch cleanup: ${{ github.event_name != 'workflow_dispatch' && true || inputs.cleanup }} - parallelism: 2 + parallelism: 1 secrets: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index dc120754be4a..118f04e91ed3 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -30,14 +30,13 @@ on: k8s_version: type: choice options: - - "1.23" - - "1.24" - "1.25" - "1.26" - "1.27" - "1.28" - "1.29" - default: "1.29" + - "1.30" + default: "1.30" cluster_name: type: string cleanup: @@ -61,7 +60,7 @@ on: required: true k8s_version: type: string - default: "1.29" + default: "1.30" enable_metrics: type: boolean default: false @@ -132,7 +131,7 @@ jobs: region: ${{ inputs.region }} cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} k8s_version: ${{ inputs.k8s_version }} - eksctl_version: v0.169.0 + eksctl_version: v0.180.0 ip_family: ${{ contains(inputs.suite, 'IPv6') && 'IPv6' || 'IPv4' }} # Set the value to IPv6 if IPv6 suite, else IPv4 private_cluster: ${{ inputs.workflow_trigger == 'private_cluster' }} git_ref: ${{ inputs.git_ref }} @@ -204,7 +203,7 @@ jobs: region: ${{ inputs.region }} cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} git_ref: ${{ inputs.git_ref }} - eksctl_version: v0.169.0 + eksctl_version: v0.180.0 private_cluster: ${{ inputs.workflow_trigger == 'private_cluster' }} - if: always() && github.event_name == 'workflow_run' uses: ./.github/actions/commit-status/end diff --git a/.github/workflows/image-canary.yaml b/.github/workflows/image-canary.yaml index 68e3c4d0bd7b..c3669c539b46 100644 --- a/.github/workflows/image-canary.yaml +++ b/.github/workflows/image-canary.yaml @@ -1,5 +1,6 @@ name: ImageCanary on: + workflow_dispatch: schedule: - cron: '0 */1 * * *' jobs: @@ -17,7 +18,7 @@ jobs: aws-region: ${{ vars.READONLY_REGION }} role-duration-seconds: 900 # Authenticate to public ECR to prevent rate limiting - - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: public.ecr.aws logout: true diff --git a/.github/workflows/postsubmit.yaml b/.github/workflows/postsubmit.yaml new file mode 100644 index 000000000000..b6ee1a63b000 --- /dev/null +++ b/.github/workflows/postsubmit.yaml @@ -0,0 +1,19 @@ +name: PostSubmit +on: + push: + branches: + - main + workflow_dispatch: +jobs: + postsubmit: + if: github.repository == 'aws/karpenter-provider-aws' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/install-deps + - name: hydrate-goproxy + run: | + mkdir -p hydrate-goproxy + cd hydrate-goproxy + go mod init hydrate-goproxy + go get "github.com/${GITHUB_REPOSITORY}@${GITHUB_SHA}" \ No newline at end of file diff --git a/.github/workflows/resource-count.yaml b/.github/workflows/resource-count.yaml index fa54688ae03c..fd643210bc1f 100644 --- a/.github/workflows/resource-count.yaml +++ b/.github/workflows/resource-count.yaml @@ -20,7 +20,7 @@ jobs: with: role-to-assume: arn:aws:iam::${{ vars.CI_ACCOUNT_ID }}:role/${{ vars.CI_ROLE_NAME }} aws-region: ${{ matrix.region }} - - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version-file: test/hack/resource/go.mod check-latest: true diff --git a/.github/workflows/snapshot.yaml b/.github/workflows/snapshot.yaml index adffc5722373..d1969b579c65 100644 --- a/.github/workflows/snapshot.yaml +++ b/.github/workflows/snapshot.yaml @@ -4,6 +4,7 @@ on: branches: - 'main' - 'release-v*' + - 'staging/*' jobs: snapshot: permissions: diff --git a/.github/workflows/sweeper.yaml b/.github/workflows/sweeper.yaml index b6b4dfd1db2c..b4994df02994 100644 --- a/.github/workflows/sweeper.yaml +++ b/.github/workflows/sweeper.yaml @@ -10,6 +10,7 @@ jobs: if: vars.CI_ACCOUNT_ID != '' || github.event_name == 'workflow_dispatch' strategy: fail-fast: false + max-parallel: 1 matrix: region: [us-east-2, us-west-2, eu-west-1, eu-north-1] runs-on: ubuntu-latest @@ -20,7 +21,7 @@ jobs: with: role-to-assume: arn:aws:iam::${{ vars.CI_ACCOUNT_ID }}:role/${{ vars.CI_ROLE_NAME }} aws-region: ${{ matrix.region }} - - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version-file: test/hack/resource/go.mod check-latest: true diff --git a/.golangci.yaml b/.golangci.yaml index 8c54aa95b166..e28f6be70d88 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,16 +1,7 @@ # See https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml run: tests: true - timeout: 5m - - skip-dirs: - - tools - - website - - hack - - charts - - designs - linters: enable: - asciicheck @@ -31,12 +22,13 @@ linters: - nilerr disable: - prealloc - linters-settings: gocyclo: min-complexity: 11 govet: - check-shadowing: true + enable-all: true + disable: + - fieldalignment revive: rules: - name: dot-imports @@ -66,6 +58,12 @@ linters-settings: issues: fix: true exclude: ['declaration of "(err|ctx)" shadows declaration at'] + exclude-dirs: + - tools + - website + - hack + - charts + - designs exclude-rules: - linters: - goheader diff --git a/ADOPTERS.md b/ADOPTERS.md index 56a37a46ca7b..52fd74551607 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -31,6 +31,7 @@ If you are open to others contacting you about your use of Karpenter on Slack, a | GlobalDots | Using Karpenter to scale Kubernetes clusters for a lot of our clients & for internal needs | `@vainkop` | [GlobalDots](https://globaldots.com) | | Grafana Labs | Using Karpenter as our Autoscaling tool on EKS | `@paulajulve`, `@logyball` | [Homepage](https://grafana.com/) & [Blog](https://grafana.com/blog/2023/11/09/how-grafana-labs-switched-to-karpenter-to-reduce-costs-and-complexities-in-amazon-eks/) | | H2O.ai | Dynamically scaling CPU and GPU nodes for AI workloads | `@Ophir Zahavi`, `@Asaf Oren` | [H2O.ai](https://h2o.ai/) | +| Homa | Using Karpenter to manage dynamically big instances and save cost effectively with disruptions | `@afreyermuth98`, `@alexbescond` | [Homa](https://www.homagames.com/) | | idealo | Scaling multi-arch IPv6 clusters hosting web and event-driven applications | `@Heiko Rothe` | [Homepage](https://www.idealo.de) | | Livspace | Replacement for cluster autoscaler on production and staging EKS clusters | `@praveen-livspace` | [Homepage](https://www.livspace.com) | | Nexxiot | Easier, Safer, Cleaner Global Transportation - Using Karpenter to manage EKS nodes | `@Alex Berger` | [Homepage](https://nexxiot.com/) | @@ -55,3 +56,4 @@ If you are open to others contacting you about your use of Karpenter on Slack, a | Next Insurance | Using Karpenter to manage the nodes in all our EKS clusters, including dev and prod, on demand and spots | `@moshebs` | [Homepage](https://www.nextinsurance.com)| | Grover Group GmbH | We use Karpenter for efficient and cost effective scaling of our nodes in all of our EKS clusters | `@suraj2410` | [Homepage](https://www.grover.com/de-en) & [Engineering Techblog](https://engineering.grover.com)| | Logz.io | Using Karpenter in all of our EKS clusters for efficient and cost effective scaling of all our K8s workloads | `@pincher95`, `@Samplify` | [Homepage](https://logz.io/)| +| X3M ads | We have been using Karpenter for (almost) all our workloads since 2023 | `@mreparaz`, `@fmansilla`, `@mrmartinez95` | [Homepage](https://x3mads.com) | diff --git a/Makefile b/Makefile index 1aa29f5e69c4..45a245f75311 100644 --- a/Makefile +++ b/Makefile @@ -45,15 +45,15 @@ ci-non-test: verify licenses vulncheck ## Runs checks other than tests run: ## Run Karpenter controller binary against your local cluster SYSTEM_NAMESPACE=${KARPENTER_NAMESPACE} \ KUBERNETES_MIN_VERSION="1.19.0-0" \ - LEADER_ELECT=false \ + DISABLE_LEADER_ELECTION=true \ DISABLE_WEBHOOK=true \ CLUSTER_NAME=${CLUSTER_NAME} \ INTERRUPTION_QUEUE=${CLUSTER_NAME} \ - FEATURE_GATES="Drift=true" \ + FEATURE_GATES="SpotToSpotConsolidation=true" \ go run ./cmd/controller/main.go test: ## Run tests - go test -v ./pkg/... \ + go test ./pkg/... \ -cover -coverprofile=coverage.out -outputdir=. -coverpkg=./... \ --ginkgo.focus="${FOCUS}" \ --ginkgo.randomize-all \ @@ -75,7 +75,7 @@ e2etests: ## Run the e2e suite against your local cluster go test \ -p 1 \ -count 1 \ - -timeout 3h \ + -timeout 3.25h \ -v \ ./suites/$(shell echo $(TEST_SUITE) | tr A-Z a-z)/... \ --ginkgo.focus="${FOCUS}" \ @@ -102,8 +102,11 @@ verify: tidy download ## Verify code. Includes dependencies, linting, formatting go generate ./... hack/boilerplate.sh cp $(KARPENTER_CORE_DIR)/pkg/apis/crds/* pkg/apis/crds + hack/validation/kubelet.sh hack/validation/requirements.sh hack/validation/labels.sh + cp pkg/apis/crds/* charts/karpenter-crd/templates + hack/mutation/conversion_webhooks_injection.sh hack/github/dependabot.sh $(foreach dir,$(MOD_DIRS),cd $(dir) && golangci-lint run $(newline)) @git diff --quiet ||\ @@ -128,7 +131,8 @@ image: ## Build the Karpenter controller images using ko build $(eval IMG_TAG=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 1 | cut -d ":" -f 2 -s)) $(eval IMG_DIGEST=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 2)) -apply: image ## Deploy the controller from the current state of your git repository into your ~/.kube/config cluster +apply: verify image ## Deploy the controller from the current state of your git repository into your ~/.kube/config cluster + kubectl apply -f ./pkg/apis/crds/ helm upgrade --install karpenter charts/karpenter --namespace ${KARPENTER_NAMESPACE} \ $(HELM_OPTS) \ --set logLevel=debug \ diff --git a/README.md b/README.md index d920323cfa32..914b68b47208 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ Come discuss Karpenter in the [#karpenter](https://kubernetes.slack.com/archives Check out the [Docs](https://karpenter.sh/docs/) to learn more. ## Talks +- 03/19/2024 [Harnessing Karpenter: Transforming Kubernetes Clusters with Argo Workflows](https://www.youtube.com/watch?v=rq57liGu0H4) +- 12/04/2023 [AWS re:Invent 2023 - Harness the power of Karpenter to scale, optimize & upgrade Kubernetes](https://www.youtube.com/watch?v=lkg_9ETHeks) - 09/08/2022 [Workload Consolidation with Karpenter](https://youtu.be/BnksdJ3oOEs) - 05/19/2022 [Scaling K8s Nodes Without Breaking the Bank or Your Sanity](https://www.youtube.com/watch?v=UBb8wbfSc34) - 03/25/2022 [Karpenter @ AWS Community Day 2022](https://youtu.be/sxDtmzbNHwE?t=3931) diff --git a/charts/karpenter-crd/Chart.yaml b/charts/karpenter-crd/Chart.yaml index 70d122d7e5ec..b0cd11fd318f 100644 --- a/charts/karpenter-crd/Chart.yaml +++ b/charts/karpenter-crd/Chart.yaml @@ -1,9 +1,9 @@ apiVersion: v2 name: karpenter-crd -description: A Helm chart for Karpenter Custom Resource Definitions (CRDs) +description: A Helm chart for Karpenter Custom Resource Definitions (CRDs). type: application -version: 0.36.0 -appVersion: 0.36.0 +version: 0.37.0 +appVersion: 0.37.0 keywords: - cluster - node diff --git a/charts/karpenter-crd/README.md b/charts/karpenter-crd/README.md new file mode 100644 index 000000000000..566d9a7efa12 --- /dev/null +++ b/charts/karpenter-crd/README.md @@ -0,0 +1,15 @@ +# karpenter-crd + +![Version: 0.36.0](https://img.shields.io/badge/Version-0.36.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.36.0](https://img.shields.io/badge/AppVersion-0.36.0-informational?style=flat-square) + +A Helm chart for Karpenter Custom Resource Definitions (CRDs). + +**Homepage:** + +## Source Code + +* + +---------------------------------------------- + +Autogenerated from chart metadata using [helm-docs](https://github.com/norwoodj/helm-docs/). diff --git a/charts/karpenter-crd/README.md.gotmpl b/charts/karpenter-crd/README.md.gotmpl new file mode 100644 index 000000000000..7991d3c23035 --- /dev/null +++ b/charts/karpenter-crd/README.md.gotmpl @@ -0,0 +1,20 @@ +{{ template "chart.header" . }} +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} + +---------------------------------------------- + +Autogenerated from chart metadata using [helm-docs](https://github.com/norwoodj/helm-docs/). diff --git a/charts/karpenter-crd/artifacthub-repo.yaml b/charts/karpenter-crd/artifacthub-repo.yaml index 194c8d2496ea..814833c19915 100644 --- a/charts/karpenter-crd/artifacthub-repo.yaml +++ b/charts/karpenter-crd/artifacthub-repo.yaml @@ -1,4 +1,4 @@ -repositoryID: fda7ffc4-4672-4218-8264-321ec3b4e3cc +repositoryID: 2cfb6f76-afe1-447f-b036-cd2e230d07d7 owners: [] # - name: awsadmin1 # email: artifacthub1@aws.com diff --git a/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml deleted file mode 120000 index 3bb741dfaf65..000000000000 --- a/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml \ No newline at end of file diff --git a/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml new file mode 100644 index 000000000000..4d81b475cb1d --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -0,0 +1,1349 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: ec2nodeclasses.karpenter.k8s.aws +spec: + group: karpenter.k8s.aws + names: + categories: + - karpenter + kind: EC2NodeClass + listKind: EC2NodeClassList + plural: ec2nodeclasses + shortNames: + - ec2nc + - ec2ncs + singular: ec2nodeclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.role + name: Role + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: EC2NodeClass is the Schema for the EC2NodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. + properties: + amiFamily: + description: |- + AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + alias is specified, this field is required. + NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + the AMIFamily() helper function + enum: + - AL2 + - AL2023 + - Bottlerocket + - Custom + - Windows2019 + - Windows2022 + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + alias: + description: |- + Alias specifies which EKS optimized AMI to select. + Each alias consists of a family and an AMI version, specified as "family@version". + Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + Note: The Windows families do **not** support version pinning, and only latest may be used. + maxLength: 30 + type: string + x-kubernetes-validations: + - message: '''alias'' is improperly formatted, must match the format ''family@version''' + rule: self.matches('^[a-zA-Z0-9]+@.+$') + - message: 'family is not supported, must be one of the following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', ''windows2022''' + rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022'] + - message: windows families may only specify version 'latest' + rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022''] ? self.split(''@'')[1] == ''latest'' : true' + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + minItems: 1 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name', 'alias'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms' + rule: '!(self.exists(x, has(x.alias)) && self.size() != 1)' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + + The following are the supported values for each volume type: + + + * gp3: 3,000-16,000 IOPS + + + * io1: 100-64,000 IOPS + + + * io2: 100-64,000 IOPS + + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: KMSKeyID (ARN) of the symmetric Key Management Service (KMS) CMK used for encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + + * gp2 and gp3: 1-16,384 + + + * io1 and io2: 4-16,384 + + + * st1 and sc1: 125-16,384 + + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store disks. + enum: + - RAID0 + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. + properties: + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled + type: string + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled + type: string + httpPutResponseHopLimit: + default: 1 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 1. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional + type: string + type: object + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in subnetSelectorTerms' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances and launch templates. + type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiSelectorTerms + - securityGroupSelectorTerms + - subnetSelectorTerms + type: object + x-kubernetes-validations: + - message: must specify exactly one of ['role', 'instanceProfile'] + rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile)) + - message: changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this. + rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile)) + - message: if set, amiFamily must be 'AL2' or 'Custom' when using an AL2 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2'') : true)' + - message: if set, amiFamily must be 'AL2023' or 'Custom' when using an AL2023 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2023'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2023'') : true)' + - message: if set, amiFamily must be 'Bottlerocket' or 'Custom' when using a Bottlerocket alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''bottlerocket'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Bottlerocket'') : true)' + - message: if set, amiFamily must be 'Windows2019' or 'Custom' when using a Windows2019 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2019'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2019'') : true)' + - message: if set, amiFamily must be 'Windows2022' or 'Custom' when using a Windows2022 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2022'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2022'') : true)' + - message: must specify amiFamily if amiSelectorTerms does not contain an alias + rule: 'self.amiSelectorTerms.exists(x, has(x.alias)) ? true : has(self.amiFamily)' + status: + description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass + properties: + amis: + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. + items: + description: AMI contains resolved AMI selector values utilized for node launch + properties: + id: + description: ID of the AMI + type: string + name: + description: Name of the AMI + type: string + requirements: + description: Requirements of the AMI to be utilized on an instance type + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + required: + - id + - requirements + type: object + type: array + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + instanceProfile: + description: InstanceProfile contains the resolved instance profile for the role + type: string + securityGroups: + description: |- + SecurityGroups contains the current Security Groups values that are available to the + cluster under the SecurityGroups selectors. + items: + description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch + properties: + id: + description: ID of the security group + type: string + name: + description: Name of the security group + type: string + required: + - id + type: object + type: array + subnets: + description: |- + Subnets contains the current Subnet values that are available to the + cluster under the subnet selectors. + items: + description: Subnet contains resolved Subnet selector values utilized for node launch + properties: + id: + description: ID of the subnet + type: string + zone: + description: The associated availability zone + type: string + zoneID: + description: The associated availability zone ID + type: string + required: + - id + - zone + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} + - name: v1beta1 + schema: + openAPIV3Schema: + description: EC2NodeClass is the Schema for the EC2NodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. + properties: + amiFamily: + description: AMIFamily is the AMI family that instances use. + enum: + - AL2 + - AL2023 + - Bottlerocket + - Ubuntu + - Custom + - Windows2019 + - Windows2022 + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name) || has(x.owner)))' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + + The following are the supported values for each volume type: + + + * gp3: 3,000-16,000 IOPS + + + * io1: 100-64,000 IOPS + + + * io2: 100-64,000 IOPS + + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: KMSKeyID (ARN) of the symmetric Key Management Service (KMS) CMK used for encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + + * gp2 and gp3: 1-16,384 + + + * io1 and io2: 4-16,384 + + + * st1 and sc1: 125-16,384 + + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store disks. + enum: + - RAID0 + type: string + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. + properties: + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled + type: string + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled + type: string + httpPutResponseHopLimit: + default: 2 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 2. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional + type: string + type: object + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in subnetSelectorTerms' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances and launch templates. + type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/managed-by + rule: self.all(k, k !='karpenter.sh/managed-by') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiFamily + - securityGroupSelectorTerms + - subnetSelectorTerms + type: object + x-kubernetes-validations: + - message: amiSelectorTerms is required when amiFamily == 'Custom' + rule: 'self.amiFamily == ''Custom'' ? self.amiSelectorTerms.size() != 0 : true' + - message: must specify exactly one of ['role', 'instanceProfile'] + rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile)) + - message: changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this. + rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile)) + status: + description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass + properties: + amis: + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. + items: + description: AMI contains resolved AMI selector values utilized for node launch + properties: + id: + description: ID of the AMI + type: string + name: + description: Name of the AMI + type: string + requirements: + description: Requirements of the AMI to be utilized on an instance type + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + required: + - id + - requirements + type: object + type: array + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + instanceProfile: + description: InstanceProfile contains the resolved instance profile for the role + type: string + securityGroups: + description: |- + SecurityGroups contains the current Security Groups values that are available to the + cluster under the SecurityGroups selectors. + items: + description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch + properties: + id: + description: ID of the security group + type: string + name: + description: Name of the security group + type: string + required: + - id + type: object + type: array + subnets: + description: |- + Subnets contains the current Subnet values that are available to the + cluster under the subnet selectors. + items: + description: Subnet contains resolved Subnet selector values utilized for node launch + properties: + id: + description: ID of the subnet + type: string + zone: + description: The associated availability zone + type: string + zoneID: + description: The associated availability zone ID + type: string + required: + - id + - zone + type: object + type: array + type: object + type: object + served: true + storage: false + subresources: + status: {} +{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace }} + port: {{ .Values.webhook.port }} +{{- end }} + diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml deleted file mode 120000 index 3f572b57547e..000000000000 --- a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../pkg/apis/crds/karpenter.sh_nodeclaims.yaml \ No newline at end of file diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml new file mode 100644 index 000000000000..d32ab39da36c --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml @@ -0,0 +1,846 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: nodeclaims.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodeClaim + listKind: NodeClaimList + plural: nodeclaims + singular: nodeclaim + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + type: string + - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.providerID + name: ID + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool + priority: 1 + type: string + - jsonPath: .spec.nodeClassRef.name + name: NodeClass + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + resources: + description: Resources models the resource requirements for the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object + type: object + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + x-kubernetes-validations: + - message: spec is immutable + rule: self == oldSelf + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + imageID: + description: ImageID is an identifier for the image that runs on the node + type: string + lastPodEventTime: + description: |- + LastPodEventTime is updated with the last time a pod was scheduled + or removed from the node. A pod going terminal or terminating + is also considered as removed. + format: date-time + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool + priority: 1 + type: string + - jsonPath: .spec.nodeClassRef.name + name: NodeClass + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + resources: + description: Resources models the resource requirements for the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object + type: object + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + required: + - nodeClassRef + - requirements + type: object + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + imageID: + description: ImageID is an identifier for the image that runs on the node + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace }} + port: {{ .Values.webhook.port }} +{{- end }} + diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml deleted file mode 120000 index 36d2d1dd918a..000000000000 --- a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml new file mode 100644 index 000000000000..f656ac273252 --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml @@ -0,0 +1,1097 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: nodepools.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodePool + listKind: NodePoolList + plural: nodepools + singular: nodepool + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .status.resources.nodes + name: Nodes + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: integer + - jsonPath: .status.resources.cpu + name: CPU + priority: 1 + type: string + - jsonPath: .status.resources.memory + name: Memory + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. + properties: + disruption: + description: Disruption contains the parameters that relate to Karpenter's disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons for disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + consolidationPolicy: + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized + type: string + required: + - consolidateAfter + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + maxProperties: 100 + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + - message: label domain "karpenter.k8s.aws" is restricted + rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + required: + - spec + type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. + properties: + disruption: + default: + consolidationPolicy: WhenUnderutilized + expireAfter: 720h + description: Disruption contains the parameters that relate to Karpenter's disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + consolidationPolicy: + default: WhenUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenUnderutilized" if not specified + enum: + - WhenEmpty + - WhenUnderutilized + type: string + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + type: object + x-kubernetes-validations: + - message: consolidateAfter cannot be combined with consolidationPolicy=WhenUnderutilized + rule: 'has(self.consolidateAfter) ? self.consolidationPolicy != ''WhenUnderutilized'' || self.consolidateAfter == ''Never'' : true' + - message: consolidateAfter must be specified with consolidationPolicy=WhenEmpty + rule: 'self.consolidationPolicy == ''WhenEmpty'' ? has(self.consolidateAfter) : true' + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + maxProperties: 100 + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + - message: label domain "karpenter.k8s.aws" is restricted + rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + resources: + description: Resources models the resource requirements for the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object + type: object + maxProperties: 0 + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + required: + - nodeClassRef + - requirements + type: object + required: + - spec + type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace }} + port: {{ .Values.webhook.port }} +{{- end }} + diff --git a/charts/karpenter-crd/values.yaml b/charts/karpenter-crd/values.yaml new file mode 100644 index 000000000000..f9c9ae3eabef --- /dev/null +++ b/charts/karpenter-crd/values.yaml @@ -0,0 +1,7 @@ +webhook: + # -- Whether to enable the webhooks and webhook permissions. + enabled: true + serviceName: karpenter + serviceNamespace: kube-system + # -- The container port to use for the webhook. + port: 8443 \ No newline at end of file diff --git a/charts/karpenter/Chart.yaml b/charts/karpenter/Chart.yaml index f82559d208f1..4ee804fca789 100644 --- a/charts/karpenter/Chart.yaml +++ b/charts/karpenter/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: karpenter description: A Helm chart for Karpenter, an open-source node provisioning project built for Kubernetes. type: application -version: 0.36.0 -appVersion: 0.36.0 +version: 0.37.0 +appVersion: 0.37.0 keywords: - cluster - node diff --git a/charts/karpenter/README.md b/charts/karpenter/README.md index a6e7a0b4883d..68f02bd816e7 100644 --- a/charts/karpenter/README.md +++ b/charts/karpenter/README.md @@ -2,7 +2,7 @@ A Helm chart for Karpenter, an open-source node provisioning project built for Kubernetes. -![Version: 0.36.0](https://img.shields.io/badge/Version-0.36.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.36.0](https://img.shields.io/badge/AppVersion-0.36.0-informational?style=flat-square) +![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.37.0](https://img.shields.io/badge/AppVersion-0.37.0-informational?style=flat-square) ## Documentation @@ -15,7 +15,7 @@ You can follow the detailed installation instruction in the [documentation](http ```bash helm upgrade --install --namespace karpenter --create-namespace \ karpenter oci://public.ecr.aws/karpenter/karpenter \ - --version 0.36.0 \ + --version 0.37.0 \ --set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN}" \ --set settings.clusterName=${CLUSTER_NAME} \ --set settings.interruptionQueue=${CLUSTER_NAME} \ @@ -27,13 +27,13 @@ helm upgrade --install --namespace karpenter --create-namespace \ As the OCI Helm chart is signed by [Cosign](https://github.com/sigstore/cosign) as part of the release process you can verify the chart before installing it by running the following command. ```shell -cosign verify public.ecr.aws/karpenter/karpenter:0.36.0 \ +cosign verify public.ecr.aws/karpenter/karpenter:0.37.0 \ --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ --certificate-identity-regexp='https://github\.com/aws/karpenter-provider-aws/\.github/workflows/release\.yaml@.+' \ --certificate-github-workflow-repository=aws/karpenter-provider-aws \ --certificate-github-workflow-name=Release \ - --certificate-github-workflow-ref=refs/tags/v0.36.0 \ - --annotations version=0.36.0 + --certificate-github-workflow-ref=refs/tags/v0.37.0 \ + --annotations version=0.37.0 ``` ## Values @@ -48,10 +48,10 @@ cosign verify public.ecr.aws/karpenter/karpenter:0.36.0 \ | controller.envFrom | list | `[]` | | | controller.extraVolumeMounts | list | `[]` | Additional volumeMounts for the controller pod. | | controller.healthProbe.port | int | `8081` | The container port to use for http health probe. | -| controller.image.digest | string | `"sha256:90adaba9e8e9f66244324ca64408a5abbfe063f8c41fbbfebf226bdda4fadd58"` | SHA256 digest of the controller image. | +| controller.image.digest | string | `"sha256:157f478f5db1fe999f5e2d27badcc742bf51cc470508b3cebe78224d0947674f"` | SHA256 digest of the controller image. | | controller.image.repository | string | `"public.ecr.aws/karpenter/controller"` | Repository path to the controller image. | -| controller.image.tag | string | `"0.36.0"` | Tag of the controller image. | -| controller.metrics.port | int | `8000` | The container port to use for metrics. | +| controller.image.tag | string | `"0.37.0"` | Tag of the controller image. | +| controller.metrics.port | int | `8080` | The container port to use for metrics. | | controller.resources | object | `{}` | Resources for the controller pod. | | controller.sidecarContainer | list | `[]` | Additional sidecarContainer config | | controller.sidecarVolumeMounts | list | `[]` | Additional volumeMounts for the sidecar - this will be added to the volume mounts on top of extraVolumeMounts | @@ -62,16 +62,9 @@ cosign verify public.ecr.aws/karpenter/karpenter:0.36.0 \ | hostNetwork | bool | `false` | Bind the pod to the host network. This is required when using a custom CNI. | | imagePullPolicy | string | `"IfNotPresent"` | Image pull policy for Docker images. | | imagePullSecrets | list | `[]` | Image pull secrets for Docker images. | -| logConfig | object | `{"enabled":false,"errorOutputPaths":["stderr"],"logEncoding":"json","logLevel":{"controller":"info","global":"info","webhook":"error"},"outputPaths":["stdout"]}` | Log configuration (Deprecated: Logging configuration will be dropped by v1, use logLevel instead) | -| logConfig.enabled | bool | `false` | Whether to enable provisioning and mounting the log ConfigMap | -| logConfig.errorOutputPaths | list | `["stderr"]` | Log errorOutputPaths - defaults to stderr only | -| logConfig.logEncoding | string | `"json"` | Log encoding - defaults to json - must be one of 'json', 'console' | -| logConfig.logLevel | object | `{"controller":"info","global":"info","webhook":"error"}` | Component-based log configuration | -| logConfig.logLevel.controller | string | `"info"` | Controller log level, defaults to 'info' | -| logConfig.logLevel.global | string | `"info"` | Global log level, defaults to 'info' | -| logConfig.logLevel.webhook | string | `"error"` | Error log level, defaults to 'error' | -| logConfig.outputPaths | list | `["stdout"]` | Log outputPaths - defaults to stdout only | +| logErrorOutputPaths | list | `["stderr"]` | Log errorOutputPaths - defaults to stderr only | | logLevel | string | `"info"` | Global log level, defaults to 'info' | +| logOutputPaths | list | `["stdout"]` | Log outputPaths - defaults to stdout only | | nameOverride | string | `""` | Overrides the chart's name. | | nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selectors to schedule the pod to nodes with labels. | | podAnnotations | object | `{}` | Additional annotations for the pod. | @@ -79,6 +72,7 @@ cosign verify public.ecr.aws/karpenter/karpenter:0.36.0 \ | podDisruptionBudget.name | string | `"karpenter"` | | | podLabels | object | `{}` | Additional labels for the pod. | | podSecurityContext | object | `{"fsGroup":65532}` | SecurityContext for the pod. | +| postInstallHook.image | string | `public.ecr.aws/bitnami/kubectl:1.30` | The image to run the post-install hook. This minimally needs to have `kubectl` installed | | priorityClassName | string | `"system-cluster-critical"` | PriorityClass name for the pod. | | replicas | int | `2` | Number of replicas. | | revisionHistoryLimit | int | `10` | The number of old ReplicaSets to retain to allow rollback. | @@ -87,27 +81,27 @@ cosign verify public.ecr.aws/karpenter/karpenter:0.36.0 \ | serviceAccount.name | string | `""` | The name of the ServiceAccount to use. If not set and create is true, a name is generated using the fullname template. | | serviceMonitor.additionalLabels | object | `{}` | Additional labels for the ServiceMonitor. | | serviceMonitor.enabled | bool | `false` | Specifies whether a ServiceMonitor should be created. | -| serviceMonitor.endpointConfig | object | `{}` | Endpoint configuration for the ServiceMonitor. | -| settings | object | `{"assumeRoleARN":"","assumeRoleDuration":"15m","batchIdleDuration":"1s","batchMaxDuration":"10s","clusterCABundle":"","clusterEndpoint":"","clusterName":"","featureGates":{"drift":true,"spotToSpotConsolidation":false},"interruptionQueue":"","isolatedVPC":false,"reservedENIs":"0","vmMemoryOverheadPercent":0.075}` | Global Settings to configure Karpenter | -| settings.assumeRoleARN | string | `""` | Role to assume for calling AWS services. | -| settings.assumeRoleDuration | string | `"15m"` | Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless assumeRoleARN set. | +| serviceMonitor.endpointConfig | object | `{}` | Configuration on `http-metrics` endpoint for the ServiceMonitor. Not to be used to add additional endpoints. See the Prometheus operator documentation for configurable fields https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint | +| settings | object | `{"batchIdleDuration":"1s","batchMaxDuration":"10s","clusterCABundle":"","clusterEndpoint":"","clusterName":"","featureGates":{"spotToSpotConsolidation":false},"interruptionQueue":"","isolatedVPC":false,"reservedENIs":"0","vmMemoryOverheadPercent":0.075}` | Global Settings to configure Karpenter | | settings.batchIdleDuration | string | `"1s"` | The maximum amount of time with no new ending pods that if exceeded ends the current batching window. If pods arrive faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods will be batched separately. | | settings.batchMaxDuration | string | `"10s"` | The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one time which usually results in fewer but larger nodes. | | settings.clusterCABundle | string | `""` | Cluster CA bundle for TLS configuration of provisioned nodes. If not set, this is taken from the controller's TLS configuration for the API server. | | settings.clusterEndpoint | string | `""` | Cluster endpoint. If not set, will be discovered during startup (EKS only) | | settings.clusterName | string | `""` | Cluster name. | -| settings.featureGates | object | `{"drift":true,"spotToSpotConsolidation":false}` | Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features | -| settings.featureGates.drift | bool | `true` | drift is in BETA and is enabled by default. Setting drift to false disables the drift disruption method to watch for drift between currently deployed nodes and the desired state of nodes set in nodepools and nodeclasses | +| settings.featureGates | object | `{"spotToSpotConsolidation":false}` | Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features | | settings.featureGates.spotToSpotConsolidation | bool | `false` | spotToSpotConsolidation is ALPHA and is disabled by default. Setting this to true will enable spot replacement consolidation for both single and multi-node consolidation. | -| settings.interruptionQueue | string | `""` | interruptionQueue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs. | +| settings.interruptionQueue | string | `""` | Interruption queue is the name of the SQS queue used for processing interruption events from EC2 Interruption handling is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs. | | settings.isolatedVPC | bool | `false` | If true then assume we can't reach AWS services which don't have a VPC endpoint This also has the effect of disabling look-ups to the AWS pricing endpoint | | settings.reservedENIs | string | `"0"` | Reserved ENIs are not included in the calculations for max-pods or kube-reserved This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html | | settings.vmMemoryOverheadPercent | float | `0.075` | The VM memory overhead as a percent that will be subtracted from the total memory for all instance types | | strategy | object | `{"rollingUpdate":{"maxUnavailable":1}}` | Strategy for updating the pod. | | terminationGracePeriodSeconds | string | `nil` | Override the default termination grace period for the pod. | | tolerations | list | `[{"key":"CriticalAddonsOnly","operator":"Exists"}]` | Tolerations to allow the pod to be scheduled to nodes with taints. | -| topologySpreadConstraints | list | `[{"maxSkew":1,"topologyKey":"topology.kubernetes.io/zone","whenUnsatisfiable":"ScheduleAnyway"}]` | Topology spread constraints to increase the controller resilience by distributing pods across the cluster zones. If an explicit label selector is not provided one will be created from the pod selector labels. | -| webhook.enabled | bool | `false` | Whether to enable the webhooks and webhook permissions. | +| topologySpreadConstraints | list | `[{"maxSkew":1,"topologyKey":"topology.kubernetes.io/zone","whenUnsatisfiable":"DoNotSchedule"}]` | Topology spread constraints to increase the controller resilience by distributing pods across the cluster zones. If an explicit label selector is not provided one will be created from the pod selector labels. | +| webhook.enabled | bool | `true` | Whether to enable the webhooks and webhook permissions. | | webhook.metrics.port | int | `8001` | The container port to use for webhook metrics. | | webhook.port | int | `8443` | The container port to use for the webhook. | +---------------------------------------------- + +Autogenerated from chart metadata using [helm-docs](https://github.com/norwoodj/helm-docs/). diff --git a/charts/karpenter/README.md.gotmpl b/charts/karpenter/README.md.gotmpl index 1e641e1ee586..ee4c8d1e4c0c 100644 --- a/charts/karpenter/README.md.gotmpl +++ b/charts/karpenter/README.md.gotmpl @@ -39,4 +39,6 @@ cosign verify public.ecr.aws/karpenter/karpenter:{{ template "chart.version" . } {{ template "chart.valuesSection" . }} -{{ template "helm-docs.versionFooter" . }} +---------------------------------------------- + +Autogenerated from chart metadata using [helm-docs](https://github.com/norwoodj/helm-docs/). diff --git a/charts/karpenter/templates/_helpers.tpl b/charts/karpenter/templates/_helpers.tpl index 9dce663e2382..8c5ffeb059d3 100644 --- a/charts/karpenter/templates/_helpers.tpl +++ b/charts/karpenter/templates/_helpers.tpl @@ -75,6 +75,17 @@ Karpenter image to use {{- end }} {{- end }} +{{/* +Karpenter post-install hook image to use +*/}} +{{- define "karpenter.postInstallHook.image" -}} +{{- if .Values.postInstallHook.image.digest }} +{{- printf "%s:%s@%s" .Values.postInstallHook.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.postInstallHook.image.tag) .Values.postInstallHook.image.digest }} +{{- else }} +{{- printf "%s:%s" .Values.postInstallHook.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.postInstallHook.image.tag) }} +{{- end }} +{{- end }} + {{/* Get PodDisruptionBudget API Version */}} {{- define "karpenter.pdb.apiVersion" -}} @@ -146,7 +157,7 @@ Flatten the stdout logging outputs from args provided */}} {{- define "karpenter.outputPathsList" -}} {{ $paths := list -}} -{{- range .Values.logConfig.outputPaths -}} +{{- range .Values.logOutputPaths -}} {{- if not (has (printf "%s" . | quote) $paths) -}} {{- $paths = printf "%s" . | quote | append $paths -}} {{- end -}} @@ -159,7 +170,7 @@ Flatten the stderr logging outputs from args provided */}} {{- define "karpenter.errorOutputPathsList" -}} {{ $paths := list -}} -{{- range .Values.logConfig.errorOutputPaths -}} +{{- range .Values.logErrorOutputPaths -}} {{- if not (has (printf "%s" . | quote) $paths) -}} {{- $paths = printf "%s" . | quote | append $paths -}} {{- end -}} diff --git a/charts/karpenter/templates/clusterrole-core.yaml b/charts/karpenter/templates/clusterrole-core.yaml index 1bce8bfcfc5a..bf39ed0c0f62 100644 --- a/charts/karpenter/templates/clusterrole-core.yaml +++ b/charts/karpenter/templates/clusterrole-core.yaml @@ -36,16 +36,20 @@ rules: resources: ["pods", "nodes", "persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses", "csinodes"] + resources: ["storageclasses", "csinodes", "volumeattachments"] verbs: ["get", "watch", "list"] - apiGroups: ["apps"] resources: ["daemonsets", "deployments", "replicasets", "statefulsets"] verbs: ["list", "watch"] -{{- if .Values.webhook.enabled }} - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations"] + {{- if .Values.webhook.enabled }} + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] verbs: ["get", "watch", "list"] -{{- end }} + {{- else }} + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get"] + {{- end }} - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["get", "list", "watch"] @@ -61,16 +65,22 @@ rules: verbs: ["create", "patch"] - apiGroups: [""] resources: ["nodes"] - verbs: ["patch", "delete"] + verbs: ["patch", "delete", "update"] - apiGroups: [""] resources: ["pods/eviction"] verbs: ["create"] -{{- if .Values.webhook.enabled }} - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations"] - verbs: ["update"] - resourceNames: ["validation.webhook.karpenter.sh", "validation.webhook.config.karpenter.sh"] -{{- end }} + - apiGroups: [""] + resources: ["pods"] + verbs: ["delete"] + {{- if .Values.webhook.enabled }} + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["update", "patch"] + {{- else }} + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["patch"] + {{- end }} {{- with .Values.additionalClusterRoleRules -}} {{ toYaml . | nindent 2 }} - {{- end -}} \ No newline at end of file + {{- end -}} diff --git a/charts/karpenter/templates/clusterrole.yaml b/charts/karpenter/templates/clusterrole.yaml index a4c08562794e..769a343d03d3 100644 --- a/charts/karpenter/templates/clusterrole.yaml +++ b/charts/karpenter/templates/clusterrole.yaml @@ -35,14 +35,4 @@ rules: # Write - apiGroups: ["karpenter.k8s.aws"] resources: ["ec2nodeclasses", "ec2nodeclasses/status"] - verbs: ["patch", "update"] -{{- if .Values.webhook.enabled }} - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations"] - verbs: ["update"] - resourceNames: ["validation.webhook.karpenter.k8s.aws"] - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["mutatingwebhookconfigurations"] - verbs: ["update"] - resourceNames: ["defaulting.webhook.karpenter.k8s.aws"] -{{- end }} + verbs: ["patch", "update"] \ No newline at end of file diff --git a/charts/karpenter/templates/configmap-logging.yaml b/charts/karpenter/templates/configmap-logging.yaml deleted file mode 100644 index d055fbc8c4cc..000000000000 --- a/charts/karpenter/templates/configmap-logging.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- if .Values.logConfig.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: config-logging - namespace: {{ .Release.Namespace }} - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -data: - # https://github.com/uber-go/zap/blob/aa3e73ec0896f8b066ddf668597a02f89628ee50/config.go - zap-logger-config: | - { - "level": "{{ .Values.logConfig.logLevel.global }}", - "development": false, - "disableStacktrace": true, - "disableCaller": true, - "sampling": { - "initial": 100, - "thereafter": 100 - }, - "outputPaths": [{{ include "karpenter.outputPathsList" . }}], - "errorOutputPaths": [{{ include "karpenter.errorOutputPathsList" . }}], - "encoding": "{{ .Values.logConfig.logEncoding }}", - "encoderConfig": { - "timeKey": "time", - "levelKey": "level", - "nameKey": "logger", - "callerKey": "caller", - "messageKey": "message", - "stacktraceKey": "stacktrace", - "levelEncoder": "capital", - "timeEncoder": "iso8601" - } - } - loglevel.controller: {{ .Values.logConfig.logLevel.controller }} - loglevel.webhook: {{ .Values.logConfig.logLevel.webhook }} -{{- end }} \ No newline at end of file diff --git a/charts/karpenter/templates/deployment.yaml b/charts/karpenter/templates/deployment.yaml index d1994b3485b9..15389c2342ce 100644 --- a/charts/karpenter/templates/deployment.yaml +++ b/charts/karpenter/templates/deployment.yaml @@ -81,9 +81,9 @@ spec: value: "{{ .Values.webhook.port }}" - name: WEBHOOK_METRICS_PORT value: "{{ .Values.webhook.metrics.port }}" - - name: DISABLE_WEBHOOK - value: "false" {{- end }} + - name: DISABLE_WEBHOOK + value: "{{ not .Values.webhook.enabled }}" {{- with .Values.logLevel }} - name: LOG_LEVEL value: "{{ . }}" @@ -103,7 +103,7 @@ spec: divisor: "0" resource: limits.memory - name: FEATURE_GATES - value: "Drift={{ .Values.settings.featureGates.drift }},SpotToSpotConsolidation={{ .Values.settings.featureGates.spotToSpotConsolidation }}" + value: "SpotToSpotConsolidation={{ .Values.settings.featureGates.spotToSpotConsolidation }}" {{- with .Values.settings.batchMaxDuration }} - name: BATCH_MAX_DURATION value: "{{ . }}" @@ -112,14 +112,6 @@ spec: - name: BATCH_IDLE_DURATION value: "{{ . }}" {{- end }} - {{- with .Values.settings.assumeRoleARN }} - - name: ASSUME_ROLE_ARN - value: "{{ . }}" - {{- end }} - {{- with .Values.settings.assumeRoleDuration }} - - name: ASSUME_ROLE_DURATION - value: "{{ . }}" - {{- end }} {{- with .Values.settings.clusterCABundle }} - name: CLUSTER_CA_BUNDLE value: "{{ . }}" @@ -186,12 +178,8 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - {{- if or (.Values.logConfig.enabled) (.Values.controller.extraVolumeMounts) }} + {{- if .Values.controller.extraVolumeMounts }} volumeMounts: - {{- if .Values.logConfig.enabled }} - - name: config-logging - mountPath: /etc/karpenter/logging - {{- end }} {{- with .Values.controller.extraVolumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -228,13 +216,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- if or (.Values.logConfig.enabled) (.Values.extraVolumes) }} + {{- if .Values.extraVolumes }} volumes: - {{- if .Values.logConfig.enabled }} - - name: config-logging - configMap: - name: config-logging - {{- end }} {{- with .Values.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/karpenter/templates/post-install-hook.yaml b/charts/karpenter/templates/post-install-hook.yaml new file mode 100644 index 000000000000..b2fd22824b8d --- /dev/null +++ b/charts/karpenter/templates/post-install-hook.yaml @@ -0,0 +1,41 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-post-install-hook + namespace: {{ .Release.Namespace }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install,post-upgrade,post-rollback + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed + {{- with .Values.additionalAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ttlSecondsAfterFinished: 0 + template: + spec: + serviceAccountName: {{ include "karpenter.serviceAccountName" . }} + restartPolicy: OnFailure + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: post-install-job + image: {{ include "karpenter.postInstallHook.image" . }} + command: + - /bin/sh + - -c + - | + {{- if .Values.webhook.enabled }} + kubectl patch customresourcedefinitions nodepools.karpenter.sh --type='merge' -p '{"spec":{"conversion":{"strategy": "Webhook", "webhook":{"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig":{"service":{"name":"{{ include "karpenter.fullname" . }}", "port": {{ .Values.webhook.port }} ,"namespace": "{{ .Release.Namespace }}"}}}}}}' + kubectl patch customresourcedefinitions nodeclaims.karpenter.sh --type='merge' -p '{"spec":{"conversion":{"strategy": "Webhook", "webhook":{"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig":{"service":{"name":"{{ include "karpenter.fullname" . }}", "port": {{ .Values.webhook.port }} ,"namespace": "{{ .Release.Namespace }}"}}}}}}' + kubectl patch customresourcedefinitions ec2nodeclasses.karpenter.k8s.aws --type='merge' -p '{"spec":{"conversion":{"strategy": "Webhook", "webhook":{"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig":{"service":{"name":"{{ include "karpenter.fullname" . }}", "port": {{ .Values.webhook.port }} ,"namespace": "{{ .Release.Namespace }}"}}}}}}' + {{- else }} + echo "disabled webhooks" + kubectl patch customresourcedefinitions nodepools.karpenter.sh --type='json' -p '[{'op': 'remove', 'path': '/spec/conversion'}]' + kubectl patch customresourcedefinitions nodeclaims.karpenter.sh --type='json' -p '[{'op': 'remove', 'path': '/spec/conversion'}]' + kubectl patch customresourcedefinitions ec2nodeclasses.karpenter.k8s.aws --type='json' -p '[{'op': 'remove', 'path': '/spec/conversion'}]' + {{- end }} + diff --git a/charts/karpenter/templates/role.yaml b/charts/karpenter/templates/role.yaml index 3862e5b3f27c..ad83a6f14a72 100644 --- a/charts/karpenter/templates/role.yaml +++ b/charts/karpenter/templates/role.yaml @@ -75,4 +75,4 @@ rules: # Write - apiGroups: ["coordination.k8s.io"] resources: ["leases"] - verbs: ["delete"] \ No newline at end of file + verbs: ["delete"] diff --git a/charts/karpenter/templates/servicemonitor.yaml b/charts/karpenter/templates/servicemonitor.yaml index 3ab5f9288650..70f07608fb2c 100644 --- a/charts/karpenter/templates/servicemonitor.yaml +++ b/charts/karpenter/templates/servicemonitor.yaml @@ -21,9 +21,9 @@ spec: matchLabels: {{- include "karpenter.selectorLabels" . | nindent 6 }} endpoints: - - port: http-metrics - path: /metrics + - port: http-metrics + path: /metrics {{- with .Values.serviceMonitor.endpointConfig }} - {{- toYaml . | nindent 4 }} + {{- toYaml . | nindent 6 }} {{- end }} {{- end -}} diff --git a/charts/karpenter/templates/webhooks-core.yaml b/charts/karpenter/templates/webhooks-core.yaml deleted file mode 100644 index 55297a1d96f6..000000000000 --- a/charts/karpenter/templates/webhooks-core.yaml +++ /dev/null @@ -1,69 +0,0 @@ -{{- if .Values.webhook.enabled }} -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation.webhook.karpenter.sh - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -webhooks: - - name: validation.webhook.karpenter.sh - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ include "karpenter.fullname" . }} - namespace: {{ .Release.Namespace }} - port: {{ .Values.webhook.port }} - failurePolicy: Fail - sideEffects: None - rules: - - apiGroups: - - karpenter.sh - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - nodeclaims - - nodeclaims/status - scope: '*' - - apiGroups: - - karpenter.sh - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - nodepools - - nodepools/status - scope: '*' ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation.webhook.config.karpenter.sh - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -webhooks: - - name: validation.webhook.config.karpenter.sh - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ include "karpenter.fullname" . }} - namespace: {{ .Release.Namespace }} - port: {{ .Values.webhook.port }} - failurePolicy: Fail - sideEffects: None - objectSelector: - matchLabels: - app.kubernetes.io/part-of: {{ template "karpenter.name" . }} -{{- end }} \ No newline at end of file diff --git a/charts/karpenter/templates/webhooks.yaml b/charts/karpenter/templates/webhooks.yaml deleted file mode 100644 index 481574349b4b..000000000000 --- a/charts/karpenter/templates/webhooks.yaml +++ /dev/null @@ -1,67 +0,0 @@ -{{- if .Values.webhook.enabled }} -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: defaulting.webhook.karpenter.k8s.aws - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -webhooks: - - name: defaulting.webhook.karpenter.k8s.aws - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ include "karpenter.fullname" . }} - namespace: {{ .Release.Namespace }} - port: {{ .Values.webhook.port }} - failurePolicy: Fail - sideEffects: None - rules: - - apiGroups: - - karpenter.k8s.aws - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - ec2nodeclasses - - ec2nodeclasses/status - scope: '*' ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation.webhook.karpenter.k8s.aws - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -webhooks: - - name: validation.webhook.karpenter.k8s.aws - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ include "karpenter.fullname" . }} - namespace: {{ .Release.Namespace }} - port: {{ .Values.webhook.port }} - failurePolicy: Fail - sideEffects: None - rules: - - apiGroups: - - karpenter.k8s.aws - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - ec2nodeclasses - - ec2nodeclasses/status - scope: '*' -{{- end }} \ No newline at end of file diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index 0d59ba4c5801..62f65a77d934 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -27,7 +27,9 @@ serviceMonitor: enabled: false # -- Additional labels for the ServiceMonitor. additionalLabels: {} - # -- Endpoint configuration for the ServiceMonitor. + # -- Configuration on `http-metrics` endpoint for the ServiceMonitor. + # Not to be used to add additional endpoints. + # See the Prometheus operator documentation for configurable fields https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint endpointConfig: {} # -- Number of replicas. replicas: 2 @@ -79,7 +81,7 @@ affinity: topologySpreadConstraints: - maxSkew: 1 topologyKey: topology.kubernetes.io/zone - whenUnsatisfiable: ScheduleAnyway + whenUnsatisfiable: DoNotSchedule # -- Tolerations to allow the pod to be scheduled to nodes with taints. tolerations: - key: CriticalAddonsOnly @@ -99,9 +101,9 @@ controller: # -- Repository path to the controller image. repository: public.ecr.aws/karpenter/controller # -- Tag of the controller image. - tag: 0.36.0 + tag: 0.37.0 # -- SHA256 digest of the controller image. - digest: sha256:90adaba9e8e9f66244324ca64408a5abbfe063f8c41fbbfebf226bdda4fadd58 + digest: sha256:157f478f5db1fe999f5e2d27badcc742bf51cc470508b3cebe78224d0947674f # -- Additional environment variables for the controller pod. env: [] # - name: AWS_REGION @@ -131,13 +133,21 @@ controller: sidecarVolumeMounts: [] metrics: # -- The container port to use for metrics. - port: 8000 + port: 8080 healthProbe: # -- The container port to use for http health probe. port: 8081 +postInstallHook: + image: + # -- Repository path to the post-install hook. This minimally needs to have `kubectl` installed + repository: public.ecr.aws/bitnami/kubectl + # -- Tag of the post-install hook image. + tag: "1.30" + # -- SHA256 digest of the post-install hook image. + digest: sha256:13a2ad1bd37ce42ee2a6f1ab0d30595f42eb7fe4a90d6ec848550524104a1ed6 webhook: # -- Whether to enable the webhooks and webhook permissions. - enabled: false + enabled: true # -- The container port to use for the webhook. port: 8443 metrics: @@ -145,26 +155,12 @@ webhook: port: 8001 # -- Global log level, defaults to 'info' logLevel: info -# -- Log configuration (Deprecated: Logging configuration will be dropped by v1, use logLevel instead) -logConfig: - # -- Whether to enable provisioning and mounting the log ConfigMap - enabled: false - # -- Log outputPaths - defaults to stdout only - outputPaths: - - stdout - # -- Log errorOutputPaths - defaults to stderr only - errorOutputPaths: - - stderr - # -- Log encoding - defaults to json - must be one of 'json', 'console' - logEncoding: json - # -- Component-based log configuration - logLevel: - # -- Global log level, defaults to 'info' - global: info - # -- Controller log level, defaults to 'info' - controller: info - # -- Error log level, defaults to 'error' - webhook: error +# -- Log outputPaths - defaults to stdout only +logOutputPaths: + - stdout +# -- Log errorOutputPaths - defaults to stderr only +logErrorOutputPaths: + - stderr # -- Global Settings to configure Karpenter settings: # -- The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one @@ -174,10 +170,6 @@ settings: # faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods # will be batched separately. batchIdleDuration: 1s - # -- Role to assume for calling AWS services. - assumeRoleARN: "" - # -- Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless assumeRoleARN set. - assumeRoleDuration: 15m # -- Cluster CA bundle for TLS configuration of provisioned nodes. If not set, this is taken from the controller's TLS configuration for the API server. clusterCABundle: "" # -- Cluster name. @@ -189,7 +181,8 @@ settings: isolatedVPC: false # -- The VM memory overhead as a percent that will be subtracted from the total memory for all instance types vmMemoryOverheadPercent: 0.075 - # -- interruptionQueue is disabled if not specified. Enabling interruption handling may + # -- Interruption queue is the name of the SQS queue used for processing interruption events from EC2 + # Interruption handling is disabled if not specified. Enabling interruption handling may # require additional permissions on the controller service account. Additional permissions are outlined in the docs. interruptionQueue: "" # -- Reserved ENIs are not included in the calculations for max-pods or kube-reserved @@ -198,10 +191,6 @@ settings: # -- Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates # in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features featureGates: - # -- drift is in BETA and is enabled by default. - # Setting drift to false disables the drift disruption method to watch for drift between currently deployed nodes - # and the desired state of nodes set in nodepools and nodeclasses - drift: true # -- spotToSpotConsolidation is ALPHA and is disabled by default. # Setting this to true will enable spot replacement consolidation for both single and multi-node consolidation. spotToSpotConsolidation: false diff --git a/cmd/controller/main.go b/cmd/controller/main.go index b1871bd91f1e..9a51e4537aef 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -24,13 +24,13 @@ import ( "sigs.k8s.io/karpenter/pkg/cloudprovider/metrics" corecontrollers "sigs.k8s.io/karpenter/pkg/controllers" - "sigs.k8s.io/karpenter/pkg/controllers/state" coreoperator "sigs.k8s.io/karpenter/pkg/operator" corewebhooks "sigs.k8s.io/karpenter/pkg/webhooks" ) func main() { ctx, op := operator.NewOperator(coreoperator.NewOperator()) + awsCloudProvider := cloudprovider.New( op.InstanceTypesProvider, op.InstanceProvider, @@ -38,22 +38,22 @@ func main() { op.GetClient(), op.AMIProvider, op.SecurityGroupProvider, - op.SubnetProvider, ) lo.Must0(op.AddHealthzCheck("cloud-provider", awsCloudProvider.LivenessProbe)) cloudProvider := metrics.Decorate(awsCloudProvider) op. WithControllers(ctx, corecontrollers.NewControllers( + op.Manager, op.Clock, op.GetClient(), - state.NewCluster(op.Clock, op.GetClient(), cloudProvider), op.EventRecorder, cloudProvider, )...). WithWebhooks(ctx, corewebhooks.NewWebhooks()...). WithControllers(ctx, controllers.NewControllers( ctx, + op.Manager, op.Session, op.Clock, op.GetClient(), @@ -67,7 +67,8 @@ func main() { op.PricingProvider, op.AMIProvider, op.LaunchTemplateProvider, + op.InstanceTypesProvider, )...). WithWebhooks(ctx, webhooks.NewWebhooks()...). - Start(ctx) + Start(ctx, cloudProvider) } diff --git a/designs/v1-api.md b/designs/v1-api.md new file mode 100644 index 000000000000..96167cd5d3b9 --- /dev/null +++ b/designs/v1-api.md @@ -0,0 +1,211 @@ +# Karpenter v1 API + +_This RFC is an extension of the [v1 API RFC](https://github.com/kubernetes-sigs/karpenter/blob/main/designs/v1-api.md) that is merged in the [`kubernetes-sigs/karpenter` repo](https://github.com/kubernetes-sigs/karpenter)._ + +## Overview + +Karpenter released the beta version of its APIs and features in October 2023. The intention behind this beta was that we would be able to determine the final set of changes and feature adds that we wanted to add to Karpenter before we considered Karpenter feature-complete. The list below details the features that Karpenter has on its roadmap before Karpenter becomes feature complete and stable at v1. + +### Categorization + +This list represents the minimal set of changes that are needed to ensure proper operational excellence, feature completeness, and stability by v1. For a change to make it on this list, it must meet one of the following criteria: + +1. Breaking: The feature requires changes or removals from the API that would be considered breaking after a bump to v1 +2. Stability: The feature ensures proper operational excellence for behavior that is leaky or has race conditions in the beta state +3. Planned Deprecations: The feature cleans-up deprecations that were previously planned the project + +## EC2NodeClass API + +``` +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass +metadata: + name: default +spec: + kubelet: + podsPerCore: 2 + maxPods: 20 + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + kubeReserved: + cpu: 200m + memory: 100Mi + ephemeral-storage: 3Gi + evictionHard: + memory.available: 5% + nodefs.available: 10% + nodefs.inodesFree: 10% + evictionSoft: + memory.available: 500Mi + nodefs.available: 15% + nodefs.inodesFree: 15% + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + evictionMaxPodGracePeriod: 60 + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + cpuCFSQuota: true + clusterDNS: ["10.0.1.100"] + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + - id: subnet-09fa4a0a8f233a921 + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + - name: my-security-group + - id: sg-063d7acfb4b06c82c + amiFamily: AL2023 + amiSelectorTerms: + - alias: al2023@v20240625 + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + - name: my-ami + - id: ami-123 + role: "KarpenterNodeRole-${CLUSTER_NAME}" + instanceProfile: "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" + userData: | + echo "Hello world" + tags: + team: team-a + app: team-a-app + instanceStorePolicy: RAID0 + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 # This is changed to disable IMDS access from containers not on the host network + httpTokens: required + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 100Gi + volumeType: gp3 + iops: 10000 + encrypted: true + kmsKeyID: "1234abcd-12ab-34cd-56ef-1234567890ab" + deleteOnTermination: true + throughput: 125 + snapshotID: snap-0123456789 + detailedMonitoring: **true** +status: + subnets: + - id: subnet-0a462d98193ff9fac + zone: us-east-2b + - id: subnet-0322dfafd76a609b6 + zone: us-east-2c + - id: subnet-0727ef01daf4ac9fe + zone: us-east-2b + - id: subnet-00c99aeafe2a70304 + zone: us-east-2a + - id: subnet-023b232fd5eb0028e + zone: us-east-2c + - id: subnet-03941e7ad6afeaa72 + zone: us-east-2a + securityGroups: + - id: sg-041513b454818610b + name: ClusterSharedNodeSecurityGroup + - id: sg-0286715698b894bca + name: ControlPlaneSecurityGroup-1AQ073TSAAPW + amis: + - id: ami-01234567890123456 + name: custom-ami-amd64 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - id: ami-01234567890123456 + name: custom-ami-arm64 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" + conditions: + - lastTransitionTime: "2024-02-02T19:54:34Z" + status: "True" + type: InstanceProfileReady + - lastTransitionTime: "2024-02-02T19:54:34Z" + status: "True" + type: SubnetsReady + - lastTransitionTime: "2024-02-02T19:54:34Z" + status: "True" + type: SecurityGroupsReady + - lastTransitionTime: "2024-02-02T19:54:34Z" + status: "True" + type: AMIsReady + - lastTransitionTime: "2024-02-02T19:54:34Z" + status: "True" + type: Ready +``` + +### Printer Columns + +**Category:** Stability, Breaking + +#### Current + +``` +➜ karpenter git:(main) ✗ k get ec2nodeclasses -o wide +NAME AGE +default 2d8h +``` + +#### Proposed + +``` +➜ karpenter git:(main) ✗ k get ec2nodeclasses -o wide +NAME READY AGE ROLE +default True 2d8h KarpenterNodeRole-test-cluster +``` + +**Standard Columns** + +1. Name +3. Ready - EC2NodeClasses now have status conditions that inform the user whether the EC2NodeClass has resolved all of its data and is “ready” to be used by a NodePool. This readiness should be easily viewable by users. +4. Age + +**Wide Columns (-o wide)** + +1. Role - As a best practice, we are recommending that users use a Node role and let Karpenter create a managed instance profile on behalf of the customer. We should easily expose this role. + +#### Status Conditions + +**Category:** Stability + +Defining the complete set of status condition types that we will include on v1 launch is **out of scope** of this document and will be defined with more granularly in Karpenter’s Observability design. Minimally for v1, we will add a `Ready` condition so that we can determine whether a EC2NodeClass can be used by a NodePool during scheduling. More robustly, we will define status conditions that ensure that each required “concept” that’s needed for an instance launch is resolved e.g. InstanceProfile resolved, Subnet resolved, Security Groups resolved, etc. + +#### Require AMISelectorTerms + +**Category:** Stability, Breaking + +When specifying AMIFamily with no AMISelectorTerms, users are currently configured to automatically update AMIs when a new version of the EKS-optimized image in that family is released. Existing nodes on older versions of the AMI will drift to the newer version to meet the desired state of the EC2NodeClass. + +This works well in pre-prod environments where it’s nice to get auto-upgraded to the latest version for testing but is extremely risky in production environments. [Karpenter now recommends to users to pin AMIs in their production environments](https://karpenter.sh/docs/tasks/managing-amis/#option-1-manage-how-amis-are-tested-and-rolled-out:~:text=The%20safest%20way%2C%20and%20the%20one%20we%20recommend%2C%20for%20ensuring%20that%20a%20new%20AMI%20doesn%E2%80%99t%20break%20your%20workloads%20is%20to%20test%20it%20before%20putting%20it%20into%20production); however, it’s still possible to be caught by surprise today that Karpenter has this behavior when you deploy a EC2NodeClass and NodePool with an AMIFamily. Most notably, this is different from eksctl and MNG, where they will get the latest AMI when you first deploy the node group, but will pin it at the point that you add it. + +We no longer want to deal with potential confusion around whether nodes will get rolled or not when using an AMIFamily with no `amiSelectorTerms`. Instead, `amiSelectorTerms` will now be required and a new term type, `alias`, will be introduced which allows users to select an EKS optimized AMI. Each alias consists of an AMI family and a version. Users can set the version to `latest` to continue to get automatic upgrades, or pin to a specific version. + +#### Disable IMDS Access from Containers by Default + +**Category:** Stability, Breaking + +The HTTPPutResponseHopLimit is [part of the instance metadata settings that are configured on the node on startup](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-options.html). This setting dictates how many hops a PUT request can take before it will be rejected by IMDS. For Kubernetes pods that live in another network namespace, this means that any pod that isn’t using `hostNetwork: true` [would need to have a HopLimit of 2 set in order to access IMDS](https://aws.amazon.com/about-aws/whats-new/2020/08/amazon-eks-supports-ec2-instance-metadata-service-v2/#:~:text=However%2C%20this%20limit%20is%20incompatible%20with%20containerized%20applications%20on%20Kubernetes%20that%20run%20in%20a%20separate%20network%20namespace%20from%20the%20instance). Opening up the node for pods to reach out to IMDS is an inherent security risk. If you are able to grab a token for IMDS, you can craft a request that gives the pod the same level of access as the instance profile which orchestrates the kubelet calls on the cluster. + +We should constrain our pods to not have access to IMDS by default to not open up users to this security risk. This new default wouldn’t affect users who have already deployed EC2NodeClasses on their cluster. It would only affect new EC2NodeClasses. + +## Labels/Annotations/Tags + +#### karpenter.sh/managed-by (EC2 Instance Tag) + +**Category:** Planned Deprecations, Breaking + +Karpenter introduced the `karpenter.sh/managed-by` tag in v0.28.0 when migrating Karpenter over to NodeClaims (called Machines at the time). This migration was marked as “completed” when it tagged the instance in EC2 with the `karpenter.sh/managed-by` tag and stored the cluster name as the value. Since we have completed the NodeClaim migration, we no longer have a need for this tag; so, we can drop it. + +This tag was only useful for scoping pod identity policies with ABAC, since it stored the cluster name in the value rather than `kubernetes.io/cluster/` which stores the cluster name in the tag key. Session tags don’t work with tag keys, so we need some tag that we can recommend users to use to create pod identity policies with ABAC using OSS Karpenter. + +Starting in v1, Karpenter would use `eks:eks-cluster-name: ` for tagging and scoping instances, volumes, primary ENIs, etc. and would use `eks:eks-cluster-arn: ` for tagging and scoping instance profiles that it creates. \ No newline at end of file diff --git a/designs/v1-roadmap.md b/designs/v1-roadmap.md new file mode 100644 index 000000000000..c30de1e15267 --- /dev/null +++ b/designs/v1-roadmap.md @@ -0,0 +1,81 @@ +# Karpenter v1 Roadmap + +_This RFC is an extension of the [v1 Roadmap RFC](https://github.com/kubernetes-sigs/karpenter/blob/main/designs/v1-roadmap.md) that is merged in the [`kubernetes-sigs/karpenter` repo](https://github.com/kubernetes-sigs/karpenter)._ + +## Overview + +Karpenter released the beta version of its APIs and features in October 2023. The intention behind this beta was that we would be able to determine the final set of changes and feature adds that we wanted to add to Karpenter before we considered Karpenter feature-complete. The list below details the features that Karpenter has on its roadmap before Karpenter becomes feature complete and stable at v1. + +### Categorization + +This list represents the minimal set of changes that are needed to ensure proper operational excellence, feature completeness, and stability by v1. For a change to make it on this list, it must meet one of the following criteria: + +1. Breaking: The feature requires changes or removals from the API that would be considered breaking after a bump to v1 +2. Stability: The feature ensures proper operational excellence for behavior that is leaky or has race conditions in the beta state +3. Planned Deprecations: The feature cleans-up deprecations that were previously planned the project + +## Roadmap + +1. [v1 APIs](./v1-api) +2. [Removing Ubuntu AMIFamily](#removing-ubuntu-amifamily) +3. [Change default TopologySpreadConstraint policy for Deployment from `ScheduleAnyways` to `DoNotSchedule`](#change-default-topologyspreadconstraint-policy-for-karpenter-deployment-from-scheduleanyways-to-donotschedule) +4. [Removing Implicit ENI Public IP Configuration](#removing-implicit-eni-public-ip-configuration) + +### v1 APIs + +**Issue Ref(s):** https://github.com/kubernetes-sigs/karpenter/issues/758, https://github.com/aws/karpenter-provider-aws/issues/5006 + +**Category:** Breaking, Stability + +For Karpenter to be considered v1, the CustomResources that are shipped with an installation of the project also need to be stable at v1. Changes to Karpenter’s API (including labels, annotations, and tags) in v1 are detailed in [Karpenter v1 API](./v1-api.md). The migration path for these changes will ensure that customers will not have to roll their nodes or manually convert their resources as they did at v1beta1. Instead, we will leverage Kubernetes [conversion webhooks](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/#webhook-conversion) to automatically convert their resources to the new schema format in code. The API groups and Kind naming will remain unchanged. + +### Removing Ubuntu AMIFamily + +**Issue Ref(s):** https://github.com/aws/karpenter-provider-aws/issues/5572 + +**Category:** Breaking + +Karpenter has supported the Ubuntu AMIFamily [since the v0.6.2 version of Karpenter](https://github.com/aws/karpenter-provider-aws/pull/1323). EKS does not have formal support for the Ubuntu AMIFamily for MNG or SMNG nodes (it's currently a third-party vendor AMI). As a result, there is no direct line-of-sight between changes in things like supported Kubernetes versions or kernel updates on the image. + +Users who still want to use Ubuntu can still use a Custom AMIFamily with amiSelectorTerms pinned to the latest Ubuntu AMI ID. They can reference `bootstrapMode: AL2` to get the same userData configuration they received before. + +#### Tasks + +- [ ] Drop the Ubuntu AMIFamily from the set of enum values in the v1 CRD +- [ ] Remove the Ubuntu bootstrapping logic from the Karpenter AMIFamily providers +- [ ] Remove the Ubuntu-specific AMIFamily documentation in the karpenter.sh documentation + +### Change default TopologySpreadConstraint policy for Deployment from `ScheduleAnyways` to `DoNotSchedule` + +**Category:** Stability, Breaking + +Karpenter ships by default with multiple replicas and leader election enabled to ensure that it can run in HA (High Availability) mode. This ensures that if a pod goes down due to an outage, the other pod is able to recover quickly by shifting the leader election over. + +Karpenter currently uses the `ScheduleAnyways` zonal topologySpreadConstraint to spread its Karpenter deployment across zones. Because this is a preference, this doesn't guarantee that pods will end up in different zones, meaning that, if there is a zonal outage, multiple replicas won't increase resiliency. + +```yaml +topologySpreadConstraints: + - labelSelector: + matchLabels: + app.kubernetes.io/instance: karpenter + app.kubernetes.io/name: karpenter + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyways +``` + +As part of v1, we are changing our default from `ScheduleAnyways` to `DoNotSchedule` to enforce stronger best practices by default to ensure that Karpenter can recover quickly in the event of a zonal outage. Users who still want the old behavior can opt back into `ScheduleAnyways` by overriding the default TopologySpreadConstraint. + +#### Tasks + +- [ ] Update Karpenter's zonal topologySpreadConstraint from `whenUnsatisfiable: ScheduleAnyways` to `whenUnsatisfiable: DoNotSchedule` + +### Removing Implicit ENI Public IP Configuration + +**Category:** Planned Deprecations, Breaking + +Karpenter currently supports checking the subnets that your instance request is attempting to launch into and explicitly configuring that `AssociatePublicIPAddress: false` when you are only launching into private subnets. This feature was supported because users had specifically requested for it in https://github.com/aws/karpenter-provider-aws/issues/3815, where users were writing deny policies on their EC2 instance launches through IRSA policies or SCP for instances that attempted to create network interfaces that associated an IP address. Now with https://github.com/aws/karpenter-provider-aws/pull/5437 merged, we have the ability to set the `associatePublicIPAddress` value explicitly on the EC2NodeClass. Users can directly set this value to `false` and we will no longer need to introspect the subnets when making instance launch requests. + +#### Tasks + +- [ ] Remove the [`CheckAnyPublicIPAssociations`](https://github.com/aws/karpenter-provider-aws/blob/ea8ea0ecb042f4143e2948d4e299e169671841fe/pkg/providers/subnet/subnet.go#L97) call in our launch template creation at v1 \ No newline at end of file diff --git a/designs/v1beta1-api.md b/designs/v1beta1-api.md index 409558a71a96..025f2e65eaaa 100644 --- a/designs/v1beta1-api.md +++ b/designs/v1beta1-api.md @@ -346,6 +346,7 @@ status: 8. `karpenter.k8s.aws/instance-cpu` 9. `karpenter.k8s.aws/instance-cpu-manufacturer` 10. `karpenter.k8s.aws/instance-memory` +11. `karpenter.k8s.aws/instance-ebs-bandwidth` 11. `karpenter.k8s.aws/instance-network-bandwidth` 12. `karpenter.k8s.aws/instance-gpu-name` 13. `karpenter.k8s.aws/instance-gpu-manufacturer` diff --git a/examples/v1beta1/100-cpu-limit.yaml b/examples/v1/100-cpu-limit.yaml similarity index 88% rename from examples/v1beta1/100-cpu-limit.yaml rename to examples/v1/100-cpu-limit.yaml index c027aaac713e..6aab622f5a19 100644 --- a/examples/v1beta1/100-cpu-limit.yaml +++ b/examples/v1/100-cpu-limit.yaml @@ -2,7 +2,7 @@ # Karpenter for this NodePool. Karpenter will not provision compute that # takes the pool over a total of 100 (virtual or physical) CPU cores. --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: limitcpu100 @@ -28,24 +28,25 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: default limits: cpu: 100 --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: default annotations: kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" spec: - amiFamily: AL2 # Amazon Linux 2 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name securityGroupSelectorTerms: - tags: - karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name \ No newline at end of file + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: al2023@latest # Amazon Linux 2023 \ No newline at end of file diff --git a/examples/v1beta1/al2-custom-ami.yaml b/examples/v1/al2-custom-ami.yaml similarity index 93% rename from examples/v1beta1/al2-custom-ami.yaml rename to examples/v1/al2-custom-ami.yaml index 5aadf58c31b8..243b3b92b0c7 100644 --- a/examples/v1beta1/al2-custom-ami.yaml +++ b/examples/v1/al2-custom-ami.yaml @@ -2,7 +2,7 @@ # AL2 AMIFamily. If your AMIs are built off https://github.com/awslabs/amazon-eks-ami and can be bootstrapped # by Karpenter, this may be a good fit for you. --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -28,11 +28,11 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: al2 --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: al2 diff --git a/examples/v1beta1/al2-custom-userdata.yaml b/examples/v1/al2-custom-userdata.yaml similarity index 91% rename from examples/v1beta1/al2-custom-userdata.yaml rename to examples/v1/al2-custom-userdata.yaml index e5bb04c20071..31c52b9b67d6 100644 --- a/examples/v1beta1/al2-custom-userdata.yaml +++ b/examples/v1/al2-custom-userdata.yaml @@ -2,7 +2,7 @@ # The UserData defined in spec.UserData needs to be in the MIME-multipart format, # and will be prepended to a Karpenter managed section that will bootstrap the kubelet. --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -28,18 +28,17 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: al2 --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: al2 annotations: kubernetes.io/description: "EC2NodeClass for running Amazon Linux 2 nodes with custom user data" spec: - amiFamily: AL2 # Amazon Linux 2 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: @@ -47,6 +46,8 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: al2023@latest # Amazon Linux 2023 userData: | MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="BOUNDARY" diff --git a/examples/v1beta1/al2-kubelet-log-query.yaml b/examples/v1/al2-kubelet-log-query.yaml similarity index 93% rename from examples/v1beta1/al2-kubelet-log-query.yaml rename to examples/v1/al2-kubelet-log-query.yaml index ecfc907937e9..52b919fc4cb9 100644 --- a/examples/v1beta1/al2-kubelet-log-query.yaml +++ b/examples/v1/al2-kubelet-log-query.yaml @@ -1,7 +1,7 @@ # This example NodePool will provision instances using the AL2 EKS-Optimized AMI # and will be prepended to a Karpenter managed section that will bootstrap the kubelet. --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -27,18 +27,17 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: al2 --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: al2 annotations: kubernetes.io/description: "EC2NodeClass for running Amazon Linux 2 nodes with user data that enables the NodeLogQuery feature gate" spec: - amiFamily: AL2 # Amazon Linux 2 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: @@ -46,6 +45,8 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: al2023@latest # Amazon Linux 2023 userData: | MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="BOUNDARY" diff --git a/examples/v1beta1/al2023-custom-userdata.yaml b/examples/v1/al2023-custom-userdata.yaml similarity index 91% rename from examples/v1beta1/al2023-custom-userdata.yaml rename to examples/v1/al2023-custom-userdata.yaml index 409792c56bee..6c57619b6a57 100644 --- a/examples/v1beta1/al2023-custom-userdata.yaml +++ b/examples/v1/al2023-custom-userdata.yaml @@ -1,6 +1,6 @@ # This example NodePool will provision instances using the AL2023 EKS-Optimized AMI. --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -26,18 +26,17 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: al2023 --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: al2023 annotations: kubernetes.io/description: "EC2NodeClass for running Amazon Linux 2023 nodes with custom user data" spec: - amiFamily: AL2023 # Amazon Linux 2023 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: @@ -45,6 +44,8 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: al2023@latest # Amazon Linux 2023 userData: | MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="//" diff --git a/examples/v1beta1/bottlerocket.yaml b/examples/v1/bottlerocket.yaml similarity index 90% rename from examples/v1beta1/bottlerocket.yaml rename to examples/v1/bottlerocket.yaml index 3330a876fd18..6ca2daf229cf 100644 --- a/examples/v1beta1/bottlerocket.yaml +++ b/examples/v1/bottlerocket.yaml @@ -1,7 +1,7 @@ # This example NodePool will provision instances # running Bottlerocket OS --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -27,18 +27,17 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: bottlerocket --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: bottlerocket annotations: kubernetes.io/description: "EC2NodeClass for running Bottlerocket nodes" spec: - amiFamily: Bottlerocket role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: @@ -46,6 +45,8 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: bottlerocket@latest # Bottlerocket blockDeviceMappings: - deviceName: /dev/xvda ebs: diff --git a/examples/v1beta1/br-custom-userdata.yaml b/examples/v1/br-custom-userdata.yaml similarity index 90% rename from examples/v1beta1/br-custom-userdata.yaml rename to examples/v1/br-custom-userdata.yaml index 99f3efb09301..3559c483a232 100644 --- a/examples/v1beta1/br-custom-userdata.yaml +++ b/examples/v1/br-custom-userdata.yaml @@ -2,7 +2,7 @@ # running Bottlerocket OS and the user data settings specified in # this EC2NodeClass will be merged into Karpenter defaults. --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -28,18 +28,17 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: bottlerocket --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: bottlerocket annotations: kubernetes.io/description: "EC2NodeClass for running Bottlerocket nodes with custom user data" spec: - amiFamily: Bottlerocket role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: @@ -47,6 +46,8 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: bottlerocket@latest # Bottlerocket userData: | [settings.kubernetes] kube-api-qps = 30 diff --git a/examples/v1/custom-family-with-alias.yaml b/examples/v1/custom-family-with-alias.yaml new file mode 100644 index 000000000000..dcafe34cb417 --- /dev/null +++ b/examples/v1/custom-family-with-alias.yaml @@ -0,0 +1,64 @@ +# This example NodePool provisions instances using an AMI that belongs to a custom AMIFamily with an Alisa +# Keep in mind, that you're in charge of bootstrapping your worker nodes. +--- +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose NodePool for generic workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + group: karpenter.k8s.aws + kind: EC2NodeClass + name: custom-family +--- +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass +metadata: + name: custom-family + annotations: + kubernetes.io/description: "EC2NodeClass for running Custom AMIFamily with custom user data that doesn't conform to the other AMIFamilies" +spec: + amiFamily: Custom + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: bottlerocket@latest + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash + echo "Running my custom set-up" + + # Have the kubelet label the node + /etc/eks/bootstrap.sh my-cluster --kubelet-extra-args='--node-labels=foo=bar' + + --BOUNDARY diff --git a/examples/v1beta1/custom-family.yaml b/examples/v1/custom-family.yaml similarity index 94% rename from examples/v1beta1/custom-family.yaml rename to examples/v1/custom-family.yaml index 92aa2725266b..f8b80dfc638e 100644 --- a/examples/v1beta1/custom-family.yaml +++ b/examples/v1/custom-family.yaml @@ -1,7 +1,7 @@ # This example NodePool provisions instances using an AMI that belongs to a custom AMIFamily # Keep in mind, that you're in charge of bootstrapping your worker nodes. --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -27,11 +27,11 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: custom-family --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: custom-family diff --git a/examples/v1beta1/general-purpose.yaml b/examples/v1/general-purpose.yaml similarity index 88% rename from examples/v1beta1/general-purpose.yaml rename to examples/v1/general-purpose.yaml index d801ae8c498d..c4986d9351a1 100644 --- a/examples/v1beta1/general-purpose.yaml +++ b/examples/v1/general-purpose.yaml @@ -1,6 +1,6 @@ # This example NodePool will provision general purpose instances --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: general-purpose @@ -26,18 +26,17 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: default --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: default annotations: kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" spec: - amiFamily: AL2 # Amazon Linux 2 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: @@ -45,3 +44,5 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: al2023@latest # Amazon Linux 2023 diff --git a/examples/v1beta1/instance-store-ephemeral-storage.yaml b/examples/v1/instance-store-ephemeral-storage.yaml similarity index 90% rename from examples/v1beta1/instance-store-ephemeral-storage.yaml rename to examples/v1/instance-store-ephemeral-storage.yaml index 7ace15c928c2..030a734b07e5 100644 --- a/examples/v1beta1/instance-store-ephemeral-storage.yaml +++ b/examples/v1/instance-store-ephemeral-storage.yaml @@ -1,7 +1,7 @@ # This example NodePool will provision AL2 instances with # local NVMe instance-store disks used for node ephemeral storage. --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -30,18 +30,17 @@ spec: operator: Gt values: ["300"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: instance-store-ephemeral-storage --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: instance-store-ephemeral-storage annotations: kubernetes.io/description: "EC2NodeClass to provision nodes with instance-store ephemeral storage" spec: - amiFamily: AL2 # Amazon Linux 2 instanceStorePolicy: "RAID0" role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: @@ -50,3 +49,5 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: al2023@latest # Amazon Linux 2023 diff --git a/examples/v1beta1/large-instances.yaml b/examples/v1/large-instances.yaml similarity index 87% rename from examples/v1beta1/large-instances.yaml rename to examples/v1/large-instances.yaml index 162262b76c43..06bf71e8e1c2 100644 --- a/examples/v1beta1/large-instances.yaml +++ b/examples/v1/large-instances.yaml @@ -1,6 +1,6 @@ # This example NodePool will avoid small instance types in the cluster --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: large-instances @@ -18,18 +18,17 @@ spec: operator: Gt values: ["8191"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: default --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: default annotations: kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" spec: - amiFamily: AL2 # Amazon Linux 2 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: @@ -37,3 +36,5 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: al2023@latest # Amazon Linux 2023 diff --git a/examples/v1beta1/node-ttls.yaml b/examples/v1/max-node-lifetime.yaml similarity index 81% rename from examples/v1beta1/node-ttls.yaml rename to examples/v1/max-node-lifetime.yaml index 6f0218d12018..f10518c42e38 100644 --- a/examples/v1beta1/node-ttls.yaml +++ b/examples/v1/max-node-lifetime.yaml @@ -2,7 +2,7 @@ # that are replaced every 7 days and drain after 1 minute # with no workloads --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -28,22 +28,22 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: default + expireAfter: 168h # expire nodes after 7 days = 7 * 24h + terminationGracePeirod: 24h # grace period after 1 day = 7 * 24h, for a max node lifetime of 8 days disruption: consolidationPolicy: WhenEmpty consolidateAfter: 60s # scale down nodes after 60 seconds without workloads (excluding daemons) - expireAfter: 168h # expire nodes after 7 days = 7 * 24h --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: default annotations: kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" spec: - amiFamily: AL2 # Amazon Linux 2 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: @@ -51,3 +51,5 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: al2023@latest # Amazon Linux 2023 diff --git a/examples/v1beta1/minValues-family.yaml b/examples/v1/min-values-family.yaml similarity index 89% rename from examples/v1beta1/minValues-family.yaml rename to examples/v1/min-values-family.yaml index 30a54e737f89..d5779d637744 100644 --- a/examples/v1beta1/minValues-family.yaml +++ b/examples/v1/min-values-family.yaml @@ -2,7 +2,7 @@ # and enforces minValues to instance families which means at least that number of unique instance # families is required by the scheduler for the NodeClaim creation. --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: spot @@ -31,22 +31,23 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: default --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: default annotations: kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" spec: - amiFamily: AL2 # Amazon Linux 2 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name securityGroupSelectorTerms: - tags: - karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name \ No newline at end of file + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: al2023@v20240625 # Amazon Linux 2023 \ No newline at end of file diff --git a/examples/v1beta1/minValues-multiple-keys.yaml b/examples/v1/min-values-multiple-keys.yaml similarity index 91% rename from examples/v1beta1/minValues-multiple-keys.yaml rename to examples/v1/min-values-multiple-keys.yaml index 1fa780147d8b..39b316e201ed 100644 --- a/examples/v1beta1/minValues-multiple-keys.yaml +++ b/examples/v1/min-values-multiple-keys.yaml @@ -2,7 +2,7 @@ # at least 2 unique instance families from [c,m,r], 5 unique instance families [eg: "m5","m5d","m5dn","c5","c5d","c4" etc], 2 unique instance types [eg: "c5.2xlarge","c4.xlarge" etc] are required by the scheduler for the NodeClaim creation. # This ensures minimum flexiblity required to schedule pods into spot nodes. --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: spot @@ -35,22 +35,23 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: default --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: default annotations: kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" spec: - amiFamily: AL2 # Amazon Linux 2 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name securityGroupSelectorTerms: - tags: - karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name \ No newline at end of file + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: al2023@v20240625 # Amazon Linux 2023 \ No newline at end of file diff --git a/examples/v1beta1/multiple-arch.yaml b/examples/v1/multiple-arch.yaml similarity index 89% rename from examples/v1beta1/multiple-arch.yaml rename to examples/v1/multiple-arch.yaml index b7c4c8a853fa..5f020fdd8db1 100644 --- a/examples/v1beta1/multiple-arch.yaml +++ b/examples/v1/multiple-arch.yaml @@ -2,7 +2,7 @@ # Karpenter will choose the NodePool that suits the workload requirements and # find an AMI automatically that matches the architecture requirements --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: amd64 @@ -28,11 +28,11 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: default --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: arm64 @@ -58,22 +58,23 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: default --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: default annotations: kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" spec: - amiFamily: AL2 # Amazon Linux 2 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name securityGroupSelectorTerms: - tags: - karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name \ No newline at end of file + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: al2023@latest # Amazon Linux 2023 \ No newline at end of file diff --git a/examples/v1beta1/multiple-ebs.yaml b/examples/v1/multiple-ebs.yaml similarity index 91% rename from examples/v1beta1/multiple-ebs.yaml rename to examples/v1/multiple-ebs.yaml index 24335cacae5f..2691f48ff2a2 100644 --- a/examples/v1beta1/multiple-ebs.yaml +++ b/examples/v1/multiple-ebs.yaml @@ -1,7 +1,7 @@ # This example NodePool will provision instances # with multiple EBS volumes attached --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -27,18 +27,17 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: multiple-ebs --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: multiple-ebs annotations: kubernetes.io/description: "EC2NodeClass to provision multiple EBS volumes to attach to new nodes" spec: - amiFamily: AL2 # Amazon Linux 2 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: @@ -46,6 +45,8 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: al2023@latest # Amazon Linux 2023 blockDeviceMappings: - deviceName: /dev/xvda ebs: diff --git a/examples/v1beta1/spot.yaml b/examples/v1/spot.yaml similarity index 88% rename from examples/v1beta1/spot.yaml rename to examples/v1/spot.yaml index 774887a3a682..c2eec39cfed4 100644 --- a/examples/v1beta1/spot.yaml +++ b/examples/v1/spot.yaml @@ -1,7 +1,7 @@ # This example will use spot instance type for all # provisioned instances --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: spot @@ -27,18 +27,17 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: default --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: default annotations: kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" spec: - amiFamily: AL2 # Amazon Linux 2 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: @@ -46,3 +45,5 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: al2023@latest # Amazon Linux 2023 diff --git a/examples/v1beta1/windows2019.yaml b/examples/v1/windows-2019.yaml similarity index 88% rename from examples/v1beta1/windows2019.yaml rename to examples/v1/windows-2019.yaml index edabbee534d6..ff0dbc222689 100644 --- a/examples/v1beta1/windows2019.yaml +++ b/examples/v1/windows-2019.yaml @@ -1,6 +1,6 @@ # This example NodePool will provision instances running Windows Server 2019 --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: windows2019 @@ -26,18 +26,17 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: windows2019 --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: windows2019 annotations: kubernetes.io/description: "Nodes running Windows Server 2019" spec: - amiFamily: Windows2019 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: @@ -45,6 +44,8 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: windows2019@latest #Windows does not support pinning metadataOptions: httpProtocolIPv6: disabled httpTokens: required \ No newline at end of file diff --git a/examples/v1beta1/windows2022.yaml b/examples/v1/windows-2022.yaml similarity index 88% rename from examples/v1beta1/windows2022.yaml rename to examples/v1/windows-2022.yaml index 24fd6ccb6857..1b213778a0dd 100644 --- a/examples/v1beta1/windows2022.yaml +++ b/examples/v1/windows-2022.yaml @@ -1,6 +1,6 @@ # This example NodePool will provision instances running Windows Server 2022 --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: windows2022 @@ -26,18 +26,17 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: windows2022 --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: windows2022 annotations: kubernetes.io/description: "Nodes running Windows Server 2022" spec: - amiFamily: Windows2022 role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name subnetSelectorTerms: - tags: @@ -45,6 +44,8 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: windows2022@latest # Windows does not support pinning metadataOptions: httpProtocolIPv6: disabled httpTokens: required \ No newline at end of file diff --git a/examples/v1beta1/windows-custom-userdata.yaml b/examples/v1/windows-custom-userdata.yaml similarity index 90% rename from examples/v1beta1/windows-custom-userdata.yaml rename to examples/v1/windows-custom-userdata.yaml index dd638dc40b6f..77cfe388dcd8 100644 --- a/examples/v1beta1/windows-custom-userdata.yaml +++ b/examples/v1/windows-custom-userdata.yaml @@ -3,7 +3,7 @@ # and they will be prepended to a Karpenter managed section that will bootstrap the kubelet. # This example also applies to the Windows 2019 EKS-Optimized AMI. --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: windows2022 @@ -29,11 +29,11 @@ spec: operator: Gt values: ["2"] nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws/v1 kind: EC2NodeClass name: windows2022 --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: windows2022 @@ -48,6 +48,8 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - alias: windows2022@latest # Windows does not support pinning metadataOptions: httpProtocolIPv6: disabled httpTokens: required diff --git a/examples/v1beta1/ubuntu-kubelet-log-query.yaml b/examples/v1beta1/ubuntu-kubelet-log-query.yaml deleted file mode 100644 index 1f5893071d73..000000000000 --- a/examples/v1beta1/ubuntu-kubelet-log-query.yaml +++ /dev/null @@ -1,71 +0,0 @@ -# This example NodePool provisions instances using the Ubuntu EKS AMI -# and will be prepended to a Karpenter managed section that will bootstrap the kubelet. ---- -apiVersion: karpenter.sh/v1beta1 -kind: NodePool -metadata: - name: ubuntu - annotations: - kubernetes.io/description: "General purpose NodePool for generic workloads" -spec: - template: - spec: - requirements: - - key: kubernetes.io/arch - operator: In - values: ["amd64"] - - key: kubernetes.io/os - operator: In - values: ["linux"] - - key: karpenter.sh/capacity-type - operator: In - values: ["on-demand"] - - key: karpenter.k8s.aws/instance-category - operator: In - values: ["c", "m", "r"] - - key: karpenter.k8s.aws/instance-generation - operator: Gt - values: ["2"] - nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 - kind: EC2NodeClass - name: ubuntu ---- -apiVersion: karpenter.k8s.aws/v1beta1 -kind: EC2NodeClass -metadata: - name: ubuntu - annotations: - kubernetes.io/description: "EC2NodeClass for running Ubuntu nodes with user data that enables the NodeLogQuery feature gate" -spec: - amiFamily: Ubuntu - role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name - subnetSelectorTerms: - - tags: - karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name - securityGroupSelectorTerms: - - tags: - karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name - userData: | - MIME-Version: 1.0 - Content-Type: multipart/mixed; boundary="BOUNDARY" - - --BOUNDARY - Content-Type: text/x-shellscript; charset="us-ascii" - - #!/bin/bash - # There is currently a bug with log query and kubelet running inside a snap environment - # https://github.com/kubernetes/kubernetes/issues/120618 - # This example is provided for reference on how to change Ubuntu settings in user data - - set -e - - # This requires Kubernetes 1.27 or above (alpha feature) - # This modifies the configuration of the /etc/eks/bootstrap.sh script because /etc/kubernetes/kubelet/kubelet-config.json - # doesn't exist before bootstrap.sh is run - - sed -i 's/args="$KUBELET_EXTRA_ARGS"/args="--feature-gates=NodeLogQuery=true $KUBELET_EXTRA_ARGS"/g' /etc/eks/bootstrap.sh - sed -i '/# writes kubeReserved and evictionHard/a echo "$(jq .enableSystemLogHandler=true $KUBELET_CONFIG)" > $KUBELET_CONFIG' /etc/eks/bootstrap.sh - sed -i '/# writes kubeReserved and evictionHard/a echo "$(jq .enableSystemLogQuery=true $KUBELET_CONFIG)" > $KUBELET_CONFIG' /etc/eks/bootstrap.sh - - --BOUNDARY-- \ No newline at end of file diff --git a/examples/workloads/arm64.yaml b/examples/workloads/arm64.yaml index 2518290abd22..26a78c58e840 100644 --- a/examples/workloads/arm64.yaml +++ b/examples/workloads/arm64.yaml @@ -3,7 +3,7 @@ kind: Deployment metadata: name: arm64 spec: - replicas: 0 + replicas: 1 selector: matchLabels: app: arm64 @@ -12,6 +12,10 @@ spec: labels: app: arm64 spec: + securityContext: + runAsUser: 2000 + runAsGroup: 3000 + fsGroup: 2000 containers: - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 name: arm64 @@ -19,5 +23,7 @@ spec: requests: cpu: "1" memory: 256M + securityContext: + allowPrivilegedEscalation: false nodeSelector: kubernetes.io/arch: arm64 \ No newline at end of file diff --git a/examples/workloads/disruption-budget.yaml b/examples/workloads/disruption-budget.yaml index 5c98cd0591e7..755aeca3e27e 100644 --- a/examples/workloads/disruption-budget.yaml +++ b/examples/workloads/disruption-budget.yaml @@ -22,6 +22,10 @@ spec: labels: app: pdb spec: + securityContext: + runAsUser: 2000 + runAsGroup: 3000 + fsGroup: 2000 containers: - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 name: pdb @@ -29,5 +33,7 @@ spec: requests: cpu: "1" memory: 256M + securityContext: + allowPrivilegedEscalation: false nodeSelector: kubernetes.io/arch: amd64 \ No newline at end of file diff --git a/examples/workloads/gpu-amd.yaml b/examples/workloads/gpu-amd.yaml index e4b53383b87f..804a7f249174 100644 --- a/examples/workloads/gpu-amd.yaml +++ b/examples/workloads/gpu-amd.yaml @@ -12,6 +12,10 @@ spec: labels: app: gpu-amd spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 containers: - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 name: gpu-amd @@ -20,4 +24,6 @@ spec: amd.com/gpu: "1" requests: cpu: "1" - memory: 256M \ No newline at end of file + memory: 256M + securityContext: + allowPrivilegedEscalation: false \ No newline at end of file diff --git a/examples/workloads/gpu-nvidia.yaml b/examples/workloads/gpu-nvidia.yaml index a995f7289cc7..f820f1c40249 100644 --- a/examples/workloads/gpu-nvidia.yaml +++ b/examples/workloads/gpu-nvidia.yaml @@ -12,6 +12,10 @@ spec: labels: app: gpu-nvidia spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 containers: - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 name: gpu-nvidia @@ -20,4 +24,6 @@ spec: nvidia.com/gpu: "1" requests: cpu: "1" - memory: 256M \ No newline at end of file + memory: 256M + securityContext: + allowPrivilegedEscalation: false \ No newline at end of file diff --git a/examples/workloads/inflate.yaml b/examples/workloads/inflate.yaml index 714aec34f7a3..8c8ef0f13823 100644 --- a/examples/workloads/inflate.yaml +++ b/examples/workloads/inflate.yaml @@ -11,6 +11,10 @@ spec: labels: app: inflate spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 containers: - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 name: inflate @@ -18,3 +22,5 @@ spec: requests: cpu: "1" memory: 256M + securityContext: + allowPrivilegeEscalation: false diff --git a/examples/workloads/neuron.yaml b/examples/workloads/neuron.yaml index 651e9ca482e7..e7cf74e13230 100644 --- a/examples/workloads/neuron.yaml +++ b/examples/workloads/neuron.yaml @@ -12,6 +12,10 @@ spec: labels: app: neuron spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 containers: - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 name: neuron @@ -20,4 +24,6 @@ spec: aws.amazon.com/neuron: "1" requests: cpu: "1" - memory: 256M \ No newline at end of file + memory: 256M + securityContext: + allowPrivilegeEscalation: false \ No newline at end of file diff --git a/examples/workloads/prefer-arm.yaml b/examples/workloads/prefer-arm.yaml index 23aaa2110cd2..382bfff85234 100644 --- a/examples/workloads/prefer-arm.yaml +++ b/examples/workloads/prefer-arm.yaml @@ -12,6 +12,10 @@ spec: labels: app: prefer-arm spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -35,4 +39,6 @@ spec: resources: requests: cpu: "1" - memory: 256M \ No newline at end of file + memory: 256M + securityContext: + allowPrivilegeEscalation: false \ No newline at end of file diff --git a/examples/workloads/spot.yaml b/examples/workloads/spot.yaml index 30cf5ec66728..409b1d7baf04 100644 --- a/examples/workloads/spot.yaml +++ b/examples/workloads/spot.yaml @@ -12,6 +12,10 @@ spec: labels: app: spot spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 containers: - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 name: spot @@ -19,5 +23,7 @@ spec: requests: cpu: "1" memory: 256M + securityContext: + allowPrivilegeEscalation: false nodeSelector: karpenter.sh/capacity-type: spot \ No newline at end of file diff --git a/examples/workloads/spread-hostname-zone.yaml b/examples/workloads/spread-hostname-zone.yaml index 106d618ae9d3..7d96b99f831c 100644 --- a/examples/workloads/spread-hostname-zone.yaml +++ b/examples/workloads/spread-hostname-zone.yaml @@ -12,6 +12,10 @@ spec: labels: app: host-zone-spread spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 containers: - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 name: host-zone-spread @@ -19,6 +23,8 @@ spec: requests: cpu: "1" memory: 256M + securityContext: + allowPrivilegeEscalation: false topologySpreadConstraints: - labelSelector: matchLabels: diff --git a/examples/workloads/spread-hostname.yaml b/examples/workloads/spread-hostname.yaml index 154ea403cc68..ce4c9fa5d8bd 100644 --- a/examples/workloads/spread-hostname.yaml +++ b/examples/workloads/spread-hostname.yaml @@ -12,6 +12,10 @@ spec: labels: app: host-spread spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 containers: - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 name: host-spread @@ -19,6 +23,8 @@ spec: requests: cpu: "1" memory: 256M + securityContext: + allowPrivilegeEscalation: false topologySpreadConstraints: - labelSelector: matchLabels: diff --git a/examples/workloads/spread-zone.yaml b/examples/workloads/spread-zone.yaml index b777dc865954..f251aacf1b7e 100644 --- a/examples/workloads/spread-zone.yaml +++ b/examples/workloads/spread-zone.yaml @@ -12,6 +12,10 @@ spec: labels: app: zone-spread spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 containers: - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 name: zone-spread @@ -19,6 +23,8 @@ spec: requests: cpu: "1" memory: 256M + securityContext: + allowPrivilegeEscalation: false topologySpreadConstraints: - labelSelector: matchLabels: diff --git a/go.mod b/go.mod index 8c46a45ea222..050b91b64788 100644 --- a/go.mod +++ b/go.mod @@ -1,35 +1,38 @@ module github.com/aws/karpenter-provider-aws -go 1.22 +go 1.22.5 require ( github.com/Pallinder/go-randomdata v1.2.0 - github.com/PuerkitoBio/goquery v1.9.1 + github.com/PuerkitoBio/goquery v1.9.2 github.com/avast/retry-go v3.0.0+incompatible - github.com/aws/aws-sdk-go v1.51.16 + github.com/aws/aws-sdk-go v1.55.5 github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20240410220356-6b868db24881 github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647 + github.com/awslabs/operatorpkg v0.0.0-20240805231134-67d0acfb6306 github.com/go-logr/zapr v1.3.0 github.com/imdario/mergo v0.3.16 + github.com/jonathan-innis/aws-sdk-go-prometheus v0.1.1-0.20240804232425-54c8227e0bab github.com/mitchellh/hashstructure/v2 v2.0.2 - github.com/onsi/ginkgo/v2 v2.17.1 - github.com/onsi/gomega v1.32.0 + github.com/onsi/ginkgo/v2 v2.20.0 + github.com/onsi/gomega v1.34.1 github.com/patrickmn/go-cache v2.1.0+incompatible - github.com/pelletier/go-toml/v2 v2.2.0 - github.com/prometheus/client_golang v1.19.0 - github.com/samber/lo v1.39.0 + github.com/pelletier/go-toml/v2 v2.2.2 + github.com/prometheus/client_golang v1.19.1 + github.com/samber/lo v1.46.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/sync v0.7.0 - golang.org/x/time v0.5.0 - k8s.io/api v0.29.3 - k8s.io/apiextensions-apiserver v0.29.3 - k8s.io/apimachinery v0.29.3 - k8s.io/client-go v0.29.3 + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 + golang.org/x/sync v0.8.0 + k8s.io/api v0.30.3 + k8s.io/apiextensions-apiserver v0.30.3 + k8s.io/apimachinery v0.30.3 + k8s.io/client-go v0.30.3 + k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20240102154912-e7106e64919e knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd - sigs.k8s.io/controller-runtime v0.17.2 - sigs.k8s.io/karpenter v0.36.0 + sigs.k8s.io/controller-runtime v0.18.4 + sigs.k8s.io/karpenter v0.37.1-0.20240812180459-92547d1f9c20 sigs.k8s.io/yaml v1.4.0 ) @@ -46,24 +49,23 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/gobuffalo/flect v1.0.2 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -79,8 +81,8 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_model v0.6.0 // indirect - github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.53.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/prometheus/statsd_exporter v0.24.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect @@ -89,13 +91,13 @@ require ( github.com/spf13/pflag v1.0.5 // indirect go.opencensus.io v0.24.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.17.0 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/sys v0.23.0 // indirect + golang.org/x/term v0.23.0 // indirect + golang.org/x/text v0.17.0 // indirect + golang.org/x/time v0.6.0 // indirect + golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/api v0.146.0 // indirect google.golang.org/appengine v1.6.8 // indirect @@ -103,15 +105,14 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect google.golang.org/grpc v1.58.3 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/cloud-provider v0.29.3 // indirect - k8s.io/component-base v0.29.3 // indirect - k8s.io/csi-translation-lib v0.29.3 // indirect - k8s.io/klog/v2 v2.120.1 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/cloud-provider v0.30.3 // indirect + k8s.io/component-base v0.30.3 // indirect + k8s.io/csi-translation-lib v0.30.3 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index 8f6f3b5951e8..210678fda946 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= -github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI= -github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= +github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= +github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -54,12 +54,14 @@ github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= -github.com/aws/aws-sdk-go v1.51.16 h1:vnWKK8KjbftEkuPX8bRj3WHsLy1uhotn0eXptpvrxJI= -github.com/aws/aws-sdk-go v1.51.16/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20240410220356-6b868db24881 h1:m9rhsGhdepdQV96tZgfy68oU75AWAjOH8u65OefTjwA= github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20240410220356-6b868db24881/go.mod h1:+Mk5k0b6HpKobxNq+B56DOhZ+I/NiPhd5MIBhQMSTSs= github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647 h1:8yRBVsjGmI7qQsPWtIrbWP+XfwHO9Wq7gdLVzjqiZFs= github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647/go.mod h1:9NafTAUHL0FlMeL6Cu5PXnMZ1q/LnC9X2emLXHsVbM8= +github.com/awslabs/operatorpkg v0.0.0-20240805231134-67d0acfb6306 h1:0dzaVod1XLEc38H4IB+KOgStoCt8RkCVI4t+XsSPrWE= +github.com/awslabs/operatorpkg v0.0.0-20240805231134-67d0acfb6306/go.mod h1:7u2ugtOiWSvqqwNlnQ8W+2TjwnSTbHoMHnR1AKpKVMA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -93,8 +95,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= -github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -113,8 +115,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -126,10 +128,8 @@ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+ github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= -github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -196,12 +196,12 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= -github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= @@ -220,6 +220,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonathan-innis/aws-sdk-go-prometheus v0.1.1-0.20240804232425-54c8227e0bab h1:3eIYcxyYhgZEmXLn6Es74ztsfW+2SFd9WR2HwfEGumk= +github.com/jonathan-innis/aws-sdk-go-prometheus v0.1.1-0.20240804232425-54c8227e0bab/go.mod h1:DDom1Ae898wsni+arqgipv+JgtDtVDmbJB5YLOQz25s= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -270,14 +272,14 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= -github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= -github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= -github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= +github.com/onsi/ginkgo/v2 v2.20.0 h1:PE84V2mHqoT1sglvHc8ZdQtPcwmvvt29WLEEO3xmdZw= +github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= -github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -293,22 +295,22 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= -github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= -github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= -github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= +github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -330,8 +332,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= -github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/samber/lo v1.46.0 h1:w8G+oaCPgz1PoCJztqymCFaKwXt+5cCXn51uPxExFfQ= +github.com/samber/lo v1.46.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -396,8 +398,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -420,8 +422,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -460,8 +462,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -469,8 +471,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= -golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -484,8 +486,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -530,14 +532,14 @@ golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -548,13 +550,13 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -599,8 +601,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -700,8 +702,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -730,24 +732,24 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= -k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= -k8s.io/apiextensions-apiserver v0.29.3 h1:9HF+EtZaVpFjStakF4yVufnXGPRppWFEQ87qnO91YeI= -k8s.io/apiextensions-apiserver v0.29.3/go.mod h1:po0XiY5scnpJfFizNGo6puNU6Fq6D70UJY2Cb2KwAVc= -k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= -k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= -k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= -k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= -k8s.io/cloud-provider v0.29.3 h1:y39hNq0lrPD1qmqQ2ykwMJGeWF9LsepVkR2a4wskwLc= -k8s.io/cloud-provider v0.29.3/go.mod h1:daDV1WkAO6pTrdsn7v8TpN/q9n75ExUC4RJDl7vlPKk= -k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo= -k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio= -k8s.io/csi-translation-lib v0.29.3 h1:GNYCE0f86K3Xkyrk7WKKwQZkJrum6QQapbOzYxZv6Mg= -k8s.io/csi-translation-lib v0.29.3/go.mod h1:snAzieA58/oiQXQZr27b0+b6/3+ZzitwI+57cUsMKKQ= -k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= -k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= +k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= +k8s.io/apiextensions-apiserver v0.30.3 h1:oChu5li2vsZHx2IvnGP3ah8Nj3KyqG3kRSaKmijhB9U= +k8s.io/apiextensions-apiserver v0.30.3/go.mod h1:uhXxYDkMAvl6CJw4lrDN4CPbONkF3+XL9cacCT44kV4= +k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= +k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= +k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= +k8s.io/cloud-provider v0.30.3 h1:SNWZmllTymOTzIPJuhtZH6il/qVi75dQARRQAm9k6VY= +k8s.io/cloud-provider v0.30.3/go.mod h1:Ax0AVdHnM7tMYnJH1Ycy4SMBD98+4zA+tboUR9eYsY8= +k8s.io/component-base v0.30.3 h1:Ci0UqKWf4oiwy8hr1+E3dsnliKnkMLZMVbWzeorlk7s= +k8s.io/component-base v0.30.3/go.mod h1:C1SshT3rGPCuNtBs14RmVD2xW0EhRSeLvBh7AGk1quA= +k8s.io/csi-translation-lib v0.30.3 h1:wBaPWnOi14/vANRIrp8pmbdx/Pgz2QRcroH7wkodezc= +k8s.io/csi-translation-lib v0.30.3/go.mod h1:3AizNZbDttVDH1RO0x1yGEQP74e9Xbfb60IBP1oWO1o= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd h1:KJXBX9dOmRTUWduHg1gnWtPGIEl+GMh8UHdrBEZgOXE= @@ -755,12 +757,12 @@ knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd/go.mod h1:36cYnaOVHkzmhgybmYX rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0= -sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= +sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= +sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/karpenter v0.36.0 h1:i82fOsFWKwnChedKsj0Hep2yrTkAjCek/aZPSMX2dW8= -sigs.k8s.io/karpenter v0.36.0/go.mod h1:fieFojxOec/l0tDmFT7R+g/Y+SGQbL9VlcYO8xb3sLo= +sigs.k8s.io/karpenter v0.37.1-0.20240812180459-92547d1f9c20 h1:HA+J1LbKlvFWUT7w7EDvVmhBeAoqqPeRbqGp8NUnLnw= +sigs.k8s.io/karpenter v0.37.1-0.20240812180459-92547d1f9c20/go.mod h1:3NLmsnHHw8p4VutpjTOPUZyhE3qH6yGTs8O94Lsu8uw= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/hack/code/bandwidth_gen/example/gp.html b/hack/code/bandwidth_gen/example/gp.html new file mode 100644 index 000000000000..fabc67a95b53 --- /dev/null +++ b/hack/code/bandwidth_gen/example/gp.html @@ -0,0 +1,9171 @@ + +General purpose instances - Amazon EC2
General purpose instances - Amazon EC2
Available sizesPlatform summaryPerformance specificationsNetwork specificationsAmazon EBS specificationsInstance store specificationsSecurity specifications

General purpose instances

General purpose instances provide a balance of compute, memory, and networking resources. + These instances are ideal for applications that use these resources in equal proportions, + such as web servers and code repositories.

+

Available sizes

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Instance typeAvailable sizes
M5m5.large | m5.xlarge | m5.2xlarge | m5.4xlarge | m5.8xlarge | m5.12xlarge | m5.16xlarge | m5.24xlarge | m5.metal
M5am5a.large | m5a.xlarge | m5a.2xlarge | m5a.4xlarge | m5a.8xlarge | m5a.12xlarge | m5a.16xlarge | m5a.24xlarge
M5adm5ad.large | m5ad.xlarge | m5ad.2xlarge | m5ad.4xlarge | m5ad.8xlarge | m5ad.12xlarge | m5ad.16xlarge | m5ad.24xlarge
M5dm5d.large | m5d.xlarge | m5d.2xlarge | m5d.4xlarge | m5d.8xlarge | m5d.12xlarge | m5d.16xlarge | m5d.24xlarge | m5d.metal
M5dnm5dn.large | m5dn.xlarge | m5dn.2xlarge | m5dn.4xlarge | m5dn.8xlarge | m5dn.12xlarge | m5dn.16xlarge | m5dn.24xlarge | m5dn.metal
M5nm5n.large | m5n.xlarge | m5n.2xlarge | m5n.4xlarge | m5n.8xlarge | m5n.12xlarge | m5n.16xlarge | m5n.24xlarge | m5n.metal
M5znm5zn.large | m5zn.xlarge | m5zn.2xlarge | m5zn.3xlarge | m5zn.6xlarge | m5zn.12xlarge | m5zn.metal
M6am6a.large | m6a.xlarge | m6a.2xlarge | m6a.4xlarge | m6a.8xlarge | m6a.12xlarge | m6a.16xlarge | m6a.24xlarge | m6a.32xlarge | m6a.48xlarge | m6a.metal
M6gm6g.medium | m6g.large | m6g.xlarge | m6g.2xlarge | m6g.4xlarge | m6g.8xlarge | m6g.12xlarge | m6g.16xlarge | m6g.metal
M6gdm6gd.medium | m6gd.large | m6gd.xlarge | m6gd.2xlarge | m6gd.4xlarge | m6gd.8xlarge | m6gd.12xlarge | m6gd.16xlarge | m6gd.metal
M6im6i.large | m6i.xlarge | m6i.2xlarge | m6i.4xlarge | m6i.8xlarge | m6i.12xlarge | m6i.16xlarge | m6i.24xlarge | m6i.32xlarge | m6i.metal
M6idm6id.large | m6id.xlarge | m6id.2xlarge | m6id.4xlarge | m6id.8xlarge | m6id.12xlarge | m6id.16xlarge | m6id.24xlarge | m6id.32xlarge | m6id.metal
M6idnm6idn.large | m6idn.xlarge | m6idn.2xlarge | m6idn.4xlarge | m6idn.8xlarge | m6idn.12xlarge | m6idn.16xlarge | m6idn.24xlarge | m6idn.32xlarge | m6idn.metal
M6inm6in.large | m6in.xlarge | m6in.2xlarge | m6in.4xlarge | m6in.8xlarge | m6in.12xlarge | m6in.16xlarge | m6in.24xlarge | m6in.32xlarge | m6in.metal
M7am7a.medium | m7a.large | m7a.xlarge | m7a.2xlarge | m7a.4xlarge | m7a.8xlarge | m7a.12xlarge | m7a.16xlarge | m7a.24xlarge | m7a.32xlarge | m7a.48xlarge | m7a.metal-48xl
M7gm7g.medium | m7g.large | m7g.xlarge | m7g.2xlarge | m7g.4xlarge | m7g.8xlarge | m7g.12xlarge | m7g.16xlarge | m7g.metal
M7gdm7gd.medium | m7gd.large | m7gd.xlarge | m7gd.2xlarge | m7gd.4xlarge | m7gd.8xlarge | m7gd.12xlarge | m7gd.16xlarge | m7gd.metal
M7im7i.large | m7i.xlarge | m7i.2xlarge | m7i.4xlarge | m7i.8xlarge | m7i.12xlarge | m7i.16xlarge | m7i.24xlarge | m7i.48xlarge | m7i.metal-24xl | m7i.metal-48xl
M7i-flexm7i-flex.large | m7i-flex.xlarge | m7i-flex.2xlarge | m7i-flex.4xlarge | m7i-flex.8xlarge
Mac1mac1.metal
Mac2mac2.metal
Mac2-m2mac2-m2.metal
Mac2-m2promac2-m2pro.metal
T2t2.nano | t2.micro | t2.small | t2.medium | t2.large | t2.xlarge | t2.2xlarge
T3t3.nano | t3.micro | t3.small | t3.medium | t3.large | t3.xlarge | t3.2xlarge
T3at3a.nano | t3a.micro | t3a.small | t3a.medium | t3a.large | t3a.xlarge | t3a.2xlarge
T4gt4g.nano | t4g.micro | t4g.small | t4g.medium | t4g.large | t4g.xlarge | t4g.2xlarge
+ +

Platform summary

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Instance typeHypervisorProcessor type (architecture)Metal instances availableDedicated Hosts supportSpot supportHibernation supportSupported operating systems
M5NitroIntel (x86_64)✓✓✓✓Windows | Linux
M5aNitroAMD (x86_64)✗✗✓✓Windows | Linux
M5adNitroAMD (x86_64)✗✗✓✓Windows | Linux
M5dNitroIntel (x86_64)✓✓✓✓Windows | Linux
M5dnNitroIntel (x86_64)✓✓✓✗Windows | Linux
M5nNitroIntel (x86_64)✓✓✓✗Windows | Linux
M5znNitroIntel (x86_64)✓✓✓✗Windows | Linux
M6aNitroAMD (x86_64)✓✓✓✗Windows | Linux
M6gNitroAWS Graviton (arm64)✓✓✓✗Linux
M6gdNitroAWS Graviton (arm64)✓✓✓✗Linux
M6iNitroIntel (x86_64)✓✓✓✓Windows | Linux
M6idNitroIntel (x86_64)✓✓✓✓Windows | Linux
M6idnNitroIntel (x86_64)✓✓✓✗Windows | Linux
M6inNitroIntel (x86_64)✓✓✓✗Windows | Linux
M7aNitroAMD (x86_64)✓✓✓✗Windows | Linux
M7gNitroAWS Graviton (arm64)✓✓✓✗Linux
M7gdNitroAWS Graviton (arm64)✓✓✓✗Linux
M7iNitroIntel (x86_64)✓✓✓✓Windows | Linux
M7i-flexNitroIntel (x86_64)✗✗✓✓Windows | Linux
Mac1NitroIntel (x86_64_mac)✓✓✗✗Linux
Mac2NitroApple (arm64_mac)✓✓✗✗Linux
Mac2-m2NitroApple (arm64_mac)✓✓✗✗Linux
Mac2-m2proNitroApple (arm64_mac)✓✓✗✗Linux
T2XenIntel (x86_64)✗✗✓✓Windows | Linux
T3NitroIntel (x86_64)✗✓✓✓Windows | Linux
T3aNitroAMD (x86_64)✗✗✓✓Windows | Linux
T4gNitroAWS Graviton (arm64)✗✗✓✗Linux
+ +

Performance specifications

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Instance typeBurstableMemory (GiB)ProcessorvCPUsCPU coresThreads per coreAccelerators
M5
m5.large✗8.00Intel Xeon Platinum 8175212✗
m5.xlarge✗16.00Intel Xeon Platinum 8175422✗
m5.2xlarge✗32.00Intel Xeon Platinum 8175842✗
m5.4xlarge✗64.00Intel Xeon Platinum 81751682✗
m5.8xlarge✗128.00Intel Xeon Platinum 817532162✗
m5.12xlarge✗192.00Intel Xeon Platinum 817548242✗
m5.16xlarge✗256.00Intel Xeon Platinum 817564322✗
m5.24xlarge✗384.00Intel Xeon Platinum 817596482✗
m5.metal✗384.00Intel Xeon Platinum 817596482✗
M5a
m5a.large✗8.00AMD EPYC 7571212✗
m5a.xlarge✗16.00AMD EPYC 7571422✗
m5a.2xlarge✗32.00AMD EPYC 7571842✗
m5a.4xlarge✗64.00AMD EPYC 75711682✗
m5a.8xlarge✗128.00AMD EPYC 757132162✗
m5a.12xlarge✗192.00AMD EPYC 757148242✗
m5a.16xlarge✗256.00AMD EPYC 757164322✗
m5a.24xlarge✗384.00AMD EPYC 757196482✗
M5ad
m5ad.large✗8.00AMD EPYC 7571212✗
m5ad.xlarge✗16.00AMD EPYC 7571422✗
m5ad.2xlarge✗32.00AMD EPYC 7571842✗
m5ad.4xlarge✗64.00AMD EPYC 75711682✗
m5ad.8xlarge✗128.00AMD EPYC 757132162✗
m5ad.12xlarge✗192.00AMD EPYC 757148242✗
m5ad.16xlarge✗256.00AMD EPYC 757164322✗
m5ad.24xlarge✗384.00AMD EPYC 757196482✗
M5d
m5d.large✗8.00Intel Xeon Platinum 8175212✗
m5d.xlarge✗16.00Intel Xeon Platinum 8175422✗
m5d.2xlarge✗32.00Intel Xeon Platinum 8175842✗
m5d.4xlarge✗64.00Intel Xeon Platinum 81751682✗
m5d.8xlarge✗128.00Intel Xeon Platinum 817532162✗
m5d.12xlarge✗192.00Intel Xeon Platinum 817548242✗
m5d.16xlarge✗256.00Intel Xeon Platinum 817564322✗
m5d.24xlarge✗384.00Intel Xeon Platinum 817596482✗
m5d.metal✗384.00Intel Xeon Platinum 817596482✗
M5dn
m5dn.large✗8.00Intel Xeon Platinum 8259212✗
m5dn.xlarge✗16.00Intel Xeon Platinum 8259422✗
m5dn.2xlarge✗32.00Intel Xeon Platinum 8259842✗
m5dn.4xlarge✗64.00Intel Xeon Platinum 82591682✗
m5dn.8xlarge✗128.00Intel Xeon Platinum 825932162✗
m5dn.12xlarge✗192.00Intel Xeon Platinum 825948242✗
m5dn.16xlarge✗256.00Intel Xeon Platinum 825964322✗
m5dn.24xlarge✗384.00Intel Xeon Platinum 825996482✗
m5dn.metal✗384.00Intel Xeon Platinum 825996482✗
M5n
m5n.large✗8.00Intel Xeon Platinum 8259212✗
m5n.xlarge✗16.00Intel Xeon Platinum 8259422✗
m5n.2xlarge✗32.00Intel Xeon Platinum 8259842✗
m5n.4xlarge✗64.00Intel Xeon Platinum 82591682✗
m5n.8xlarge✗128.00Intel Xeon Platinum 825932162✗
m5n.12xlarge✗192.00Intel Xeon Platinum 825948242✗
m5n.16xlarge✗256.00Intel Xeon Platinum 825964322✗
m5n.24xlarge✗384.00Intel Xeon Platinum 825996482✗
m5n.metal✗384.00Intel Xeon Platinum 825996482✗
M5zn
m5zn.large✗8.00Intel Xeon Platinum 8252212✗
m5zn.xlarge✗16.00Intel Xeon Platinum 8252422✗
m5zn.2xlarge✗32.00Intel Xeon Platinum 8252842✗
m5zn.3xlarge✗48.00Intel Xeon Platinum 82521262✗
m5zn.6xlarge✗96.00Intel Xeon Platinum 825224122✗
m5zn.12xlarge✗192.00Intel Xeon Platinum 825248242✗
m5zn.metal✗192.00Intel Xeon Platinum 825248242✗
M6a
m6a.large✗8.00AMD EPYC 7R13212✗
m6a.xlarge✗16.00AMD EPYC 7R13422✗
m6a.2xlarge✗32.00AMD EPYC 7R13842✗
m6a.4xlarge✗64.00AMD EPYC 7R131682✗
m6a.8xlarge✗128.00AMD EPYC 7R1332162✗
m6a.12xlarge✗192.00AMD EPYC 7R1348242✗
m6a.16xlarge✗256.00AMD EPYC 7R1364322✗
m6a.24xlarge✗384.00AMD EPYC 7R1396482✗
m6a.32xlarge✗512.00AMD EPYC 7R13128642✗
m6a.48xlarge✗768.00AMD EPYC 7R13192962✗
m6a.metal✗768.00AMD EPYC 7R13192962✗
M6g
m6g.medium✗4.00AWS Graviton2 Processor111✗
m6g.large✗8.00AWS Graviton2 Processor221✗
m6g.xlarge✗16.00AWS Graviton2 Processor441✗
m6g.2xlarge✗32.00AWS Graviton2 Processor881✗
m6g.4xlarge✗64.00AWS Graviton2 Processor16161✗
m6g.8xlarge✗128.00AWS Graviton2 Processor32321✗
m6g.12xlarge✗192.00AWS Graviton2 Processor48481✗
m6g.16xlarge✗256.00AWS Graviton2 Processor64641✗
m6g.metal✗256.00AWS Graviton2 Processor64641✗
M6gd
m6gd.medium✗4.00AWS Graviton2 Processor111✗
m6gd.large✗8.00AWS Graviton2 Processor221✗
m6gd.xlarge✗16.00AWS Graviton2 Processor441✗
m6gd.2xlarge✗32.00AWS Graviton2 Processor881✗
m6gd.4xlarge✗64.00AWS Graviton2 Processor16161✗
m6gd.8xlarge✗128.00AWS Graviton2 Processor32321✗
m6gd.12xlarge✗192.00AWS Graviton2 Processor48481✗
m6gd.16xlarge✗256.00AWS Graviton2 Processor64641✗
m6gd.metal✗256.00AWS Graviton2 Processor64641✗
M6i
m6i.large✗8.00Intel Xeon Ice Lake212✗
m6i.xlarge✗16.00Intel Xeon Ice Lake422✗
m6i.2xlarge✗32.00Intel Xeon Ice Lake842✗
m6i.4xlarge✗64.00Intel Xeon Ice Lake1682✗
m6i.8xlarge✗128.00Intel Xeon Ice Lake32162✗
m6i.12xlarge✗192.00Intel Xeon Ice Lake48242✗
m6i.16xlarge✗256.00Intel Xeon Ice Lake64322✗
m6i.24xlarge✗384.00Intel Xeon Ice Lake96482✗
m6i.32xlarge✗512.00Intel Xeon Ice Lake128642✗
m6i.metal✗512.00Intel Xeon Ice Lake128642✗
M6id
m6id.large✗8.00Intel Xeon Ice Lake212✗
m6id.xlarge✗16.00Intel Xeon Ice Lake422✗
m6id.2xlarge✗32.00Intel Xeon Ice Lake842✗
m6id.4xlarge✗64.00Intel Xeon Ice Lake1682✗
m6id.8xlarge✗128.00Intel Xeon Ice Lake32162✗
m6id.12xlarge✗192.00Intel Xeon Ice Lake48242✗
m6id.16xlarge✗256.00Intel Xeon Ice Lake64322✗
m6id.24xlarge✗384.00Intel Xeon Ice Lake96482✗
m6id.32xlarge✗512.00Intel Xeon Ice Lake128642✗
m6id.metal✗512.00Intel Xeon Ice Lake128642✗
M6idn
m6idn.large✗8.00Intel Xeon Ice Lake212✗
m6idn.xlarge✗16.00Intel Xeon Ice Lake422✗
m6idn.2xlarge✗32.00Intel Xeon Ice Lake842✗
m6idn.4xlarge✗64.00Intel Xeon Ice Lake1682✗
m6idn.8xlarge✗128.00Intel Xeon Ice Lake32162✗
m6idn.12xlarge✗192.00Intel Xeon Ice Lake48242✗
m6idn.16xlarge✗256.00Intel Xeon Ice Lake64322✗
m6idn.24xlarge✗384.00Intel Xeon Ice Lake96482✗
m6idn.32xlarge✗512.00Intel Xeon Ice Lake128642✗
m6idn.metal✗512.00Intel Xeon Ice Lake128642✗
M6in
m6in.large✗8.00Intel Xeon Ice Lake212✗
m6in.xlarge✗16.00Intel Xeon Ice Lake422✗
m6in.2xlarge✗32.00Intel Xeon Ice Lake842✗
m6in.4xlarge✗64.00Intel Xeon Ice Lake1682✗
m6in.8xlarge✗128.00Intel Xeon Ice Lake32162✗
m6in.12xlarge✗192.00Intel Xeon Ice Lake48242✗
m6in.16xlarge✗256.00Intel Xeon Ice Lake64322✗
m6in.24xlarge✗384.00Intel Xeon Ice Lake96482✗
m6in.32xlarge✗512.00Intel Xeon Ice Lake128642✗
m6in.metal✗512.00Intel Xeon Ice Lake128642✗
M7a
m7a.medium✗4.00AMD EPYC 9R14111✗
m7a.large✗8.00AMD EPYC 9R14221✗
m7a.xlarge✗16.00AMD EPYC 9R14441✗
m7a.2xlarge✗32.00AMD EPYC 9R14881✗
m7a.4xlarge✗64.00AMD EPYC 9R1416161✗
m7a.8xlarge✗128.00AMD EPYC 9R1432321✗
m7a.12xlarge✗192.00AMD EPYC 9R1448481✗
m7a.16xlarge✗256.00AMD EPYC 9R1464641✗
m7a.24xlarge✗384.00AMD EPYC 9R1496961✗
m7a.32xlarge✗512.00AMD EPYC 9R141281281✗
m7a.48xlarge✗768.00AMD EPYC 9R141921921✗
m7a.metal-48xl✗768.00AMD EPYC 9R141921921✗
M7g
m7g.medium✗4.00AWS Graviton3 Processor111✗
m7g.large✗8.00AWS Graviton3 Processor221✗
m7g.xlarge✗16.00AWS Graviton3 Processor441✗
m7g.2xlarge✗32.00AWS Graviton3 Processor881✗
m7g.4xlarge✗64.00AWS Graviton3 Processor16161✗
m7g.8xlarge✗128.00AWS Graviton3 Processor32321✗
m7g.12xlarge✗192.00AWS Graviton3 Processor48481✗
m7g.16xlarge✗256.00AWS Graviton3 Processor64641✗
m7g.metal✗256.00AWS Graviton3 Processor64641✗
M7gd
m7gd.medium✗4.00AWS Graviton3 Processor111✗
m7gd.large✗8.00AWS Graviton3 Processor221✗
m7gd.xlarge✗16.00AWS Graviton3 Processor441✗
m7gd.2xlarge✗32.00AWS Graviton3 Processor881✗
m7gd.4xlarge✗64.00AWS Graviton3 Processor16161✗
m7gd.8xlarge✗128.00AWS Graviton3 Processor32321✗
m7gd.12xlarge✗192.00AWS Graviton3 Processor48481✗
m7gd.16xlarge✗256.00AWS Graviton3 Processor64641✗
m7gd.metal✗256.00AWS Graviton3 Processor64641✗
M7i
m7i.large✗8.00Intel Xeon Sapphire Rapids212✗
m7i.xlarge✗16.00Intel Xeon Sapphire Rapids422✗
m7i.2xlarge✗32.00Intel Xeon Sapphire Rapids842✗
m7i.4xlarge✗64.00Intel Xeon Sapphire Rapids1682✗
m7i.8xlarge✗128.00Intel Xeon Sapphire Rapids32162✗
m7i.12xlarge✗192.00Intel Xeon Sapphire Rapids48242✗
m7i.16xlarge✗256.00Intel Xeon Sapphire Rapids64322✗
m7i.24xlarge✗384.00Intel Xeon Sapphire Rapids96482✗
m7i.48xlarge✗768.00Intel Xeon Sapphire Rapids192962✗
m7i.metal-24xl✗384.00Intel Xeon Sapphire Rapids96482✗
m7i.metal-48xl✗768.00Intel Xeon Sapphire Rapids192962✗
M7i-flex
m7i-flex.large✗8.00Intel Xeon Sapphire Rapids212✗
m7i-flex.xlarge✗16.00Intel Xeon Sapphire Rapids422✗
m7i-flex.2xlarge✗32.00Intel Xeon Sapphire Rapids842✗
m7i-flex.4xlarge✗64.00Intel Xeon Sapphire Rapids1682✗
m7i-flex.8xlarge✗128.00Intel Xeon Sapphire Rapids32162✗
Mac1
mac1.metal✗32.00Intel Core i7-8700B1262✗
Mac2
mac2.metal✗16.00Apple M1 chip with 8-core CPU842✗
Mac2-m2
mac2-m2.metal✗24.00Apple M2 with 8‑core CPU881✗
Mac2-m2pro
mac2-m2pro.metal✗32.00Apple M2 Pro with 12‑core CPU12121✗
T2
t2.nano✓0.50Intel Xeon Family111✗
t2.micro✓1.00Intel Xeon Family111✗
t2.small✓2.00Intel Xeon Family111✗
t2.medium✓4.00Intel Broadwell E5-2686v4221✗
t2.large✓8.00Intel Broadwell E5-2686v4221✗
t2.xlarge✓16.00Intel Broadwell E5-2686v4441✗
t2.2xlarge✓32.00Intel Broadwell E5-2686v4881✗
T3
t3.nano✓0.50Intel Skylake P-8175212✗
t3.micro✓1.00Intel Skylake P-8175212✗
t3.small✓2.00Intel Skylake P-8175212✗
t3.medium✓4.00Intel Skylake P-8175212✗
t3.large✓8.00Intel Skylake P-8175212✗
t3.xlarge✓16.00Intel Skylake P-8175422✗
t3.2xlarge✓32.00Intel Skylake P-8175842✗
T3a
t3a.nano✓0.50AMD EPYC 7571212✗
t3a.micro✓1.00AMD EPYC 7571212✗
t3a.small✓2.00AMD EPYC 7571212✗
t3a.medium✓4.00AMD EPYC 7571212✗
t3a.large✓8.00AMD EPYC 7571212✗
t3a.xlarge✓16.00AMD EPYC 7571422✗
t3a.2xlarge✓32.00AMD EPYC 7571842✗
T4g
t4g.nano✓0.50AWS Graviton2 Processor221✗
t4g.micro✓1.00AWS Graviton2 Processor221✗
t4g.small✓2.00AWS Graviton2 Processor221✗
t4g.medium✓4.00AWS Graviton2 Processor221✗
t4g.large✓8.00AWS Graviton2 Processor221✗
t4g.xlarge✓16.00AWS Graviton2 Processor441✗
t4g.2xlarge✓32.00AWS Graviton2 Processor881✗
+ +

Network specifications

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Instance typeBaseline / Burst bandwidth (Gbps)EFAENAENA ExpressNetwork cardsMax. network interfacesIP addresses per interfaceIPv6
M5
m5.large 10.75 / 10.0✗✓✗1310✓
m5.xlarge 11.25 / 10.0✗✓✗1415✓
m5.2xlarge 12.5 / 10.0✗✓✗1415✓
m5.4xlarge 15.0 / 10.0✗✓✗1830✓
m5.8xlarge10 Gigabit✗✓✗1830✓
m5.12xlarge12 Gigabit✗✓✗1830✓
m5.16xlarge20 Gigabit✗✓✗11550✓
m5.24xlarge25 Gigabit✗✓✗11550✓
m5.metal25 Gigabit✗✓✗11550✓
M5a
m5a.large 10.75 / 10.0✗✓✗1310✓
m5a.xlarge 11.25 / 10.0✗✓✗1415✓
m5a.2xlarge 12.5 / 10.0✗✓✗1415✓
m5a.4xlarge 15.0 / 10.0✗✓✗1830✓
m5a.8xlarge 17.5 / 10.0✗✓✗1830✓
m5a.12xlarge10 Gigabit✗✓✗1830✓
m5a.16xlarge12 Gigabit✗✓✗11550✓
m5a.24xlarge20 Gigabit✗✓✗11550✓
M5ad
m5ad.large 10.75 / 10.0✗✓✗1310✓
m5ad.xlarge 11.25 / 10.0✗✓✗1415✓
m5ad.2xlarge 12.5 / 10.0✗✓✗1415✓
m5ad.4xlarge 15.0 / 10.0✗✓✗1830✓
m5ad.8xlarge 17.5 / 10.0✗✓✗1830✓
m5ad.12xlarge10 Gigabit✗✓✗1830✓
m5ad.16xlarge12 Gigabit✗✓✗11550✓
m5ad.24xlarge20 Gigabit✗✓✗11550✓
M5d
m5d.large 10.75 / 10.0✗✓✗1310✓
m5d.xlarge 11.25 / 10.0✗✓✗1415✓
m5d.2xlarge 12.5 / 10.0✗✓✗1415✓
m5d.4xlarge 15.0 / 10.0✗✓✗1830✓
m5d.8xlarge10 Gigabit✗✓✗1830✓
m5d.12xlarge12 Gigabit✗✓✗1830✓
m5d.16xlarge20 Gigabit✗✓✗11550✓
m5d.24xlarge25 Gigabit✗✓✗11550✓
m5d.metal25 Gigabit✗✓✗11550✓
M5dn
m5dn.large 12.1 / 25.0✗✓✗1310✓
m5dn.xlarge 14.1 / 25.0✗✓✗1415✓
m5dn.2xlarge 18.125 / 25.0✗✓✗1415✓
m5dn.4xlarge 116.25 / 25.0✗✓✗1830✓
m5dn.8xlarge25 Gigabit✗✓✗1830✓
m5dn.12xlarge50 Gigabit✗✓✗1830✓
m5dn.16xlarge75 Gigabit✗✓✗11550✓
m5dn.24xlarge100 Gigabit✓✓✗11550✓
m5dn.metal100 Gigabit✓✓✗11550✓
M5n
m5n.large 12.1 / 25.0✗✓✗1310✓
m5n.xlarge 14.1 / 25.0✗✓✗1415✓
m5n.2xlarge 18.125 / 25.0✗✓✗1415✓
m5n.4xlarge 116.25 / 25.0✗✓✗1830✓
m5n.8xlarge25 Gigabit✗✓✗1830✓
m5n.12xlarge50 Gigabit✗✓✗1830✓
m5n.16xlarge75 Gigabit✗✓✗11550✓
m5n.24xlarge100 Gigabit✓✓✗11550✓
m5n.metal100 Gigabit✓✓✗11550✓
M5zn
m5zn.large 13.0 / 25.0✗✓✗1310✓
m5zn.xlarge 15.0 / 25.0✗✓✗1415✓
m5zn.2xlarge 110.0 / 25.0✗✓✗1415✓
m5zn.3xlarge 115.0 / 25.0✗✓✗1830✓
m5zn.6xlarge50 Gigabit✗✓✗1830✓
m5zn.12xlarge100 Gigabit✓✓✗11550✓
m5zn.metal100 Gigabit✓✓✗11550✓
M6a
m6a.large 10.781 / 12.5✗✓✗1310✓
m6a.xlarge 11.562 / 12.5✗✓✗1415✓
m6a.2xlarge 13.125 / 12.5✗✓✗1415✓
m6a.4xlarge 16.25 / 12.5✗✓✗1830✓
m6a.8xlarge12.5 Gigabit✗✓✗1830✓
m6a.12xlarge18.75 Gigabit✗✓✗1830✓
m6a.16xlarge25 Gigabit✗✓✗11550✓
m6a.24xlarge37.5 Gigabit✗✓✗11550✓
m6a.32xlarge50 Gigabit✗✓✗11550✓
m6a.48xlarge50 Gigabit✓✓✓11550✓
m6a.metal50 Gigabit✓✓✓11550✓
M6g
m6g.medium 10.5 / 10.0✗✓✗124✓
m6g.large 10.75 / 10.0✗✓✗1310✓
m6g.xlarge 11.25 / 10.0✗✓✗1415✓
m6g.2xlarge 12.5 / 10.0✗✓✗1415✓
m6g.4xlarge 15.0 / 10.0✗✓✗1830✓
m6g.8xlarge12 Gigabit✗✓✗1830✓
m6g.12xlarge20 Gigabit✗✓✗1830✓
m6g.16xlarge25 Gigabit✗✓✗11550✓
m6g.metal25 Gigabit✗✓✗11550✓
M6gd
m6gd.medium 10.5 / 10.0✗✓✗124✓
m6gd.large 10.75 / 10.0✗✓✗1310✓
m6gd.xlarge 11.25 / 10.0✗✓✗1415✓
m6gd.2xlarge 12.5 / 10.0✗✓✗1415✓
m6gd.4xlarge 15.0 / 10.0✗✓✗1830✓
m6gd.8xlarge12 Gigabit✗✓✗1830✓
m6gd.12xlarge20 Gigabit✗✓✗1830✓
m6gd.16xlarge25 Gigabit✗✓✗11550✓
m6gd.metal25 Gigabit✗✓✗11550✓
M6i
m6i.large 10.781 / 12.5✗✓✗1310✓
m6i.xlarge 11.562 / 12.5✗✓✗1415✓
m6i.2xlarge 13.125 / 12.5✗✓✗1415✓
m6i.4xlarge 16.25 / 12.5✗✓✗1830✓
m6i.8xlarge12.5 Gigabit✗✓✓1830✓
m6i.12xlarge18.75 Gigabit✗✓✓1830✓
m6i.16xlarge25 Gigabit✗✓✓11550✓
m6i.24xlarge37.5 Gigabit✗✓✓11550✓
m6i.32xlarge50 Gigabit✓✓✓11550✓
m6i.metal50 Gigabit✓✓✓11550✓
M6id
m6id.large 10.781 / 12.5✗✓✗1310✓
m6id.xlarge 11.562 / 12.5✗✓✗1415✓
m6id.2xlarge 13.125 / 12.5✗✓✗1415✓
m6id.4xlarge 16.25 / 12.5✗✓✗1830✓
m6id.8xlarge12.5 Gigabit✗✓✓1830✓
m6id.12xlarge18.75 Gigabit✗✓✓1830✓
m6id.16xlarge25 Gigabit✗✓✓11550✓
m6id.24xlarge37.5 Gigabit✗✓✓11550✓
m6id.32xlarge50 Gigabit✓✓✓11550✓
m6id.metal50 Gigabit✓✓✓11550✓
M6idn
m6idn.large 13.125 / 25.0✗✓✗1310✓
m6idn.xlarge 16.25 / 30.0✗✓✗1415✓
m6idn.2xlarge 112.5 / 40.0✗✓✗1415✓
m6idn.4xlarge 125.0 / 50.0✗✓✗1830✓
m6idn.8xlarge50 Gigabit✗✓✗1830✓
m6idn.12xlarge75 Gigabit✗✓✗1830✓
m6idn.16xlarge100 Gigabit✗✓✗11550✓
m6idn.24xlarge150 Gigabit✗✓✗11550✓
m6idn.32xlarge200 Gigabit✓✓✗21650✓
m6idn.metal200 Gigabit✓✓✗21650✓
M6in
m6in.large 13.125 / 25.0✗✓✗1310✓
m6in.xlarge 16.25 / 30.0✗✓✗1415✓
m6in.2xlarge 112.5 / 40.0✗✓✗1415✓
m6in.4xlarge 125.0 / 50.0✗✓✗1830✓
m6in.8xlarge50 Gigabit✗✓✗1830✓
m6in.12xlarge75 Gigabit✗✓✗1830✓
m6in.16xlarge100 Gigabit✗✓✗11550✓
m6in.24xlarge150 Gigabit✗✓✗11550✓
m6in.32xlarge200 Gigabit✓✓✗21650✓
m6in.metal200 Gigabit✓✓✗21650✓
M7a
m7a.medium 10.39 / 12.5✗✓✗124✓
m7a.large 10.781 / 12.5✗✓✗1310✓
m7a.xlarge 11.562 / 12.5✗✓✗1415✓
m7a.2xlarge 13.125 / 12.5✗✓✗1415✓
m7a.4xlarge 16.25 / 12.5✗✓✗1830✓
m7a.8xlarge12.5 Gigabit✗✓✗1830✓
m7a.12xlarge18.75 Gigabit✗✓✗1830✓
m7a.16xlarge25 Gigabit✗✓✗11550✓
m7a.24xlarge37.5 Gigabit✗✓✗11550✓
m7a.32xlarge50 Gigabit✗✓✗11550✓
m7a.48xlarge50 Gigabit✓✓✗11550✓
m7a.metal-48xl50 Gigabit✓✓✗11550✓
M7g
m7g.medium 10.52 / 12.5✗✓✗124✓
m7g.large 10.937 / 12.5✗✓✗1310✓
m7g.xlarge 11.876 / 12.5✗✓✗1415✓
m7g.2xlarge 13.75 / 15.0✗✓✗1415✓
m7g.4xlarge 17.5 / 15.0✗✓✗1830✓
m7g.8xlarge15 Gigabit✗✓✗1830✓
m7g.12xlarge22.5 Gigabit✗✓✓1830✓
m7g.16xlarge30 Gigabit✓✓✓11550✓
m7g.metal30 Gigabit✓✓✓11550✓
M7gd
m7gd.medium 10.52 / 12.5✗✓✗124✓
m7gd.large 10.937 / 12.5✗✓✗1310✓
m7gd.xlarge 11.876 / 12.5✗✓✗1415✓
m7gd.2xlarge 13.75 / 15.0✗✓✗1415✓
m7gd.4xlarge 17.5 / 15.0✗✓✗1830✓
m7gd.8xlarge15 Gigabit✗✓✗1830✓
m7gd.12xlarge22.5 Gigabit✗✓✓1830✓
m7gd.16xlarge30 Gigabit✓✓✓11550✓
m7gd.metal30 Gigabit✓✓✓11550✓
M7i
m7i.large 10.781 / 12.5✗✓✗1310✓
m7i.xlarge 11.562 / 12.5✗✓✗1415✓
m7i.2xlarge 13.125 / 12.5✗✓✗1415✓
m7i.4xlarge 16.25 / 12.5✗✓✗1830✓
m7i.8xlarge12.5 Gigabit✗✓✗1830✓
m7i.12xlarge18.75 Gigabit✗✓✓1830✓
m7i.16xlarge25 Gigabit✗✓✓11550✓
m7i.24xlarge37.5 Gigabit✗✓✓11550✓
m7i.48xlarge50 Gigabit✓✓✓11550✓
m7i.metal-24xl37.5 Gigabit✗✓✓11550✓
m7i.metal-48xl50 Gigabit✓✓✓11550✓
M7i-flex
m7i-flex.large 10.39 / 12.5✗✓✗1310✓
m7i-flex.xlarge 10.781 / 12.5✗✓✗1415✓
m7i-flex.2xlarge 11.562 / 12.5✗✓✗1415✓
m7i-flex.4xlarge 13.125 / 12.5✗✓✗1830✓
m7i-flex.8xlarge 16.25 / 12.5✗✓✗1830✓
Mac1
mac1.metal25 Gigabit✗✓✗1830✓
Mac2
mac2.metal10 Gigabit✗✓✗1830✓
Mac2-m2
mac2-m2.metal10 Gigabit✗✓✗1830✓
Mac2-m2pro
mac2-m2pro.metal10 Gigabit✗✓✗1830✓
T2
t2.nanoLow to Moderate✗✗✗122✓
t2.microLow to Moderate✗✗✗122✓
t2.smallLow to Moderate✗✗✗134✓
t2.mediumLow to Moderate✗✗✗136✓
t2.largeLow to Moderate✗✗✗1312✓
t2.xlargeModerate✗✗✗1315✓
t2.2xlargeModerate✗✗✗1315✓
T3
t3.nano 10.032 / 5.0✗✓✗122✓
t3.micro 10.064 / 5.0✗✓✗122✓
t3.small 10.128 / 5.0✗✓✗134✓
t3.medium 10.256 / 5.0✗✓✗136✓
t3.large 10.512 / 5.0✗✓✗1312✓
t3.xlarge 11.024 / 5.0✗✓✗1415✓
t3.2xlarge 12.048 / 5.0✗✓✗1415✓
T3a
t3a.nano 10.032 / 5.0✗✓✗122✓
t3a.micro 10.064 / 5.0✗✓✗122✓
t3a.small 10.128 / 5.0✗✓✗124✓
t3a.medium 10.256 / 5.0✗✓✗136✓
t3a.large 10.512 / 5.0✗✓✗1312✓
t3a.xlarge 11.024 / 5.0✗✓✗1415✓
t3a.2xlarge 12.048 / 5.0✗✓✗1415✓
T4g
t4g.nano 10.032 / 5.0✗✓✗122✓
t4g.micro 10.064 / 5.0✗✓✗122✓
t4g.small 10.128 / 5.0✗✓✗134✓
t4g.medium 10.256 / 5.0✗✓✗136✓
t4g.large 10.512 / 5.0✗✓✗1312✓
t4g.xlarge 11.024 / 5.0✗✓✗1415✓
t4g.2xlarge 12.048 / 5.0✗✓✗1415✓
+
Note

1 These instances have a baseline bandwidth and can + use a network I/O credit mechanism to burst beyond their baseline bandwidth on a best effort basis. + Other instances types can sustain their maximum performance indefinitely. For more information, + see + instance network bandwidth.

For 32xlarge and metal instance types that + support 200 Gbps, at least 2 ENIs, each attached to a different network card, are required on the instance to achieve + 200 Gbps throughput. Each ENI attached to a network card can achieve a max of 170 Gbps.

+ +

Amazon EBS specifications

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Instance typeBaseline / Maximum bandwidth (Mbps)Baseline / Maximum throughput (MB/s, 128 KiB I/O)Baseline / Maximum IOPS (16 KiB I/O)NVMeEBS optimization 2
M5
m5.large 1650.00 / 4750.0081.25 / 593.753600.00 / 18750.00✓default
m5.xlarge 11150.00 / 4750.00143.75 / 593.756000.00 / 18750.00✓default
m5.2xlarge 12300.00 / 4750.00287.50 / 593.7512000.00 / 18750.00✓default
m5.4xlarge4750.00593.7518750.00✓default
m5.8xlarge6800.00850.0030000.00✓default
m5.12xlarge9500.001187.5040000.00✓default
m5.16xlarge13600.001700.0060000.00✓default
m5.24xlarge19000.002375.0080000.00✓default
m5.metal19000.002375.0080000.00✓default
M5a
m5a.large 1650.00 / 2880.0081.25 / 360.003600.00 / 16000.00✓default
m5a.xlarge 11085.00 / 2880.00135.62 / 360.006000.00 / 16000.00✓default
m5a.2xlarge 11580.00 / 2880.00197.50 / 360.008333.00 / 16000.00✓default
m5a.4xlarge2880.00360.0016000.00✓default
m5a.8xlarge4750.00593.7520000.00✓default
m5a.12xlarge6780.00847.5030000.00✓default
m5a.16xlarge9500.001187.5040000.00✓default
m5a.24xlarge13750.001718.7560000.00✓default
M5ad
m5ad.large 1650.00 / 2880.0081.25 / 360.003600.00 / 16000.00✓default
m5ad.xlarge 11085.00 / 2880.00135.62 / 360.006000.00 / 16000.00✓default
m5ad.2xlarge 11580.00 / 2880.00197.50 / 360.008333.00 / 16000.00✓default
m5ad.4xlarge2880.00360.0016000.00✓default
m5ad.8xlarge4750.00593.7520000.00✓default
m5ad.12xlarge6780.00847.5030000.00✓default
m5ad.16xlarge9500.001187.5040000.00✓default
m5ad.24xlarge13750.001718.7560000.00✓default
M5d
m5d.large 1650.00 / 4750.0081.25 / 593.753600.00 / 18750.00✓default
m5d.xlarge 11150.00 / 4750.00143.75 / 593.756000.00 / 18750.00✓default
m5d.2xlarge 12300.00 / 4750.00287.50 / 593.7512000.00 / 18750.00✓default
m5d.4xlarge4750.00593.7518750.00✓default
m5d.8xlarge6800.00850.0030000.00✓default
m5d.12xlarge9500.001187.5040000.00✓default
m5d.16xlarge13600.001700.0060000.00✓default
m5d.24xlarge19000.002375.0080000.00✓default
m5d.metal19000.002375.0080000.00✓default
M5dn
m5dn.large 1650.00 / 4750.0081.25 / 593.753600.00 / 18750.00✓default
m5dn.xlarge 11150.00 / 4750.00143.75 / 593.756000.00 / 18750.00✓default
m5dn.2xlarge 12300.00 / 4750.00287.50 / 593.7512000.00 / 18750.00✓default
m5dn.4xlarge4750.00593.7518750.00✓default
m5dn.8xlarge6800.00850.0030000.00✓default
m5dn.12xlarge9500.001187.5040000.00✓default
m5dn.16xlarge13600.001700.0060000.00✓default
m5dn.24xlarge19000.002375.0080000.00✓default
m5dn.metal19000.002375.0080000.00✓default
M5n
m5n.large 1650.00 / 4750.0081.25 / 593.753600.00 / 18750.00✓default
m5n.xlarge 11150.00 / 4750.00143.75 / 593.756000.00 / 18750.00✓default
m5n.2xlarge 12300.00 / 4750.00287.50 / 593.7512000.00 / 18750.00✓default
m5n.4xlarge4750.00593.7518750.00✓default
m5n.8xlarge6800.00850.0030000.00✓default
m5n.12xlarge9500.001187.5040000.00✓default
m5n.16xlarge13600.001700.0060000.00✓default
m5n.24xlarge19000.002375.0080000.00✓default
m5n.metal19000.002375.0080000.00✓default
M5zn
m5zn.large 1800.00 / 3170.00100.00 / 396.253333.00 / 13333.00✓default
m5zn.xlarge 11564.00 / 3170.00195.50 / 396.256667.00 / 13333.00✓default
m5zn.2xlarge3170.00396.2513333.00✓default
m5zn.3xlarge4750.00593.7520000.00✓default
m5zn.6xlarge9500.001187.5040000.00✓default
m5zn.12xlarge19000.002375.0080000.00✓default
m5zn.metal19000.002375.0080000.00✓default
M6a
m6a.large 1650.00 / 10000.0081.25 / 1250.003600.00 / 40000.00✓default
m6a.xlarge 11250.00 / 10000.00156.25 / 1250.006000.00 / 40000.00✓default
m6a.2xlarge 12500.00 / 10000.00312.50 / 1250.0012000.00 / 40000.00✓default
m6a.4xlarge 15000.00 / 10000.00625.00 / 1250.0020000.00 / 40000.00✓default
m6a.8xlarge10000.001250.0040000.00✓default
m6a.12xlarge15000.001875.0060000.00✓default
m6a.16xlarge20000.002500.0080000.00✓default
m6a.24xlarge30000.003750.00120000.00✓default
m6a.32xlarge40000.005000.00160000.00✓default
m6a.48xlarge40000.005000.00240000.00✓default
m6a.metal40000.005000.00240000.00✓default
M6g
m6g.medium 1315.00 / 4750.0039.38 / 593.752500.00 / 20000.00✓default
m6g.large 1630.00 / 4750.0078.75 / 593.753600.00 / 20000.00✓default
m6g.xlarge 11188.00 / 4750.00148.50 / 593.756000.00 / 20000.00✓default
m6g.2xlarge 12375.00 / 4750.00296.88 / 593.7512000.00 / 20000.00✓default
m6g.4xlarge4750.00593.7520000.00✓default
m6g.8xlarge9500.001187.5040000.00✓default
m6g.12xlarge14250.001781.2550000.00✓default
m6g.16xlarge19000.002375.0080000.00✓default
m6g.metal19000.002375.0080000.00✓default
M6gd
m6gd.medium 1315.00 / 4750.0039.38 / 593.752500.00 / 20000.00✓default
m6gd.large 1630.00 / 4750.0078.75 / 593.753600.00 / 20000.00✓default
m6gd.xlarge 11188.00 / 4750.00148.50 / 593.756000.00 / 20000.00✓default
m6gd.2xlarge 12375.00 / 4750.00296.88 / 593.7512000.00 / 20000.00✓default
m6gd.4xlarge4750.00593.7520000.00✓default
m6gd.8xlarge9500.001187.5040000.00✓default
m6gd.12xlarge14250.001781.2550000.00✓default
m6gd.16xlarge19000.002375.0080000.00✓default
m6gd.metal19000.002375.0080000.00✓default
M6i
m6i.large 1650.00 / 10000.0081.25 / 1250.003600.00 / 40000.00✓default
m6i.xlarge 11250.00 / 10000.00156.25 / 1250.006000.00 / 40000.00✓default
m6i.2xlarge 12500.00 / 10000.00312.50 / 1250.0012000.00 / 40000.00✓default
m6i.4xlarge 15000.00 / 10000.00625.00 / 1250.0020000.00 / 40000.00✓default
m6i.8xlarge10000.001250.0040000.00✓default
m6i.12xlarge15000.001875.0060000.00✓default
m6i.16xlarge20000.002500.0080000.00✓default
m6i.24xlarge30000.003750.00120000.00✓default
m6i.32xlarge40000.005000.00160000.00✓default
m6i.metal40000.005000.00160000.00✓default
M6id
m6id.large 1650.00 / 10000.0081.25 / 1250.003600.00 / 40000.00✓default
m6id.xlarge 11250.00 / 10000.00156.25 / 1250.006000.00 / 40000.00✓default
m6id.2xlarge 12500.00 / 10000.00312.50 / 1250.0012000.00 / 40000.00✓default
m6id.4xlarge 15000.00 / 10000.00625.00 / 1250.0020000.00 / 40000.00✓default
m6id.8xlarge10000.001250.0040000.00✓default
m6id.12xlarge15000.001875.0060000.00✓default
m6id.16xlarge20000.002500.0080000.00✓default
m6id.24xlarge30000.003750.00120000.00✓default
m6id.32xlarge40000.005000.00160000.00✓default
m6id.metal40000.005000.00160000.00✓default
M6idn
m6idn.large 11562.00 / 25000.00195.31 / 3125.006250.00 / 100000.00✓default
m6idn.xlarge 13125.00 / 25000.00390.62 / 3125.0012500.00 / 100000.00✓default
m6idn.2xlarge 16250.00 / 25000.00781.25 / 3125.0025000.00 / 100000.00✓default
m6idn.4xlarge 112500.00 / 25000.001562.50 / 3125.0050000.00 / 100000.00✓default
m6idn.8xlarge25000.003125.00100000.00✓default
m6idn.12xlarge37500.004687.50150000.00✓default
m6idn.16xlarge50000.006250.00200000.00✓default
m6idn.24xlarge75000.009375.00300000.00✓default
m6idn.32xlarge100000.0012500.00400000.00✓default
m6idn.metal100000.0012500.00400000.00✓default
M6in
m6in.large 11562.00 / 25000.00195.31 / 3125.006250.00 / 100000.00✓default
m6in.xlarge 13125.00 / 25000.00390.62 / 3125.0012500.00 / 100000.00✓default
m6in.2xlarge 16250.00 / 25000.00781.25 / 3125.0025000.00 / 100000.00✓default
m6in.4xlarge 112500.00 / 25000.001562.50 / 3125.0050000.00 / 100000.00✓default
m6in.8xlarge25000.003125.00100000.00✓default
m6in.12xlarge37500.004687.50150000.00✓default
m6in.16xlarge50000.006250.00200000.00✓default
m6in.24xlarge75000.009375.00300000.00✓default
m6in.32xlarge100000.0012500.00400000.00✓default
m6in.metal100000.0012500.00400000.00✓default
M7a
m7a.medium 1325.00 / 10000.0040.62 / 1250.002500.00 / 40000.00✓default
m7a.large 1650.00 / 10000.0081.25 / 1250.003600.00 / 40000.00✓default
m7a.xlarge 11250.00 / 10000.00156.25 / 1250.006000.00 / 40000.00✓default
m7a.2xlarge 12500.00 / 10000.00312.50 / 1250.0012000.00 / 40000.00✓default
m7a.4xlarge 15000.00 / 10000.00625.00 / 1250.0020000.00 / 40000.00✓default
m7a.8xlarge10000.001250.0040000.00✓default
m7a.12xlarge15000.001875.0060000.00✓default
m7a.16xlarge20000.002500.0080000.00✓default
m7a.24xlarge30000.003750.00120000.00✓default
m7a.32xlarge40000.005000.00160000.00✓default
m7a.48xlarge40000.005000.00240000.00✓default
m7a.metal-48xl40000.005000.00240000.00✓default
M7g
m7g.medium 1315.00 / 10000.0039.38 / 1250.002500.00 / 40000.00✓default
m7g.large 1630.00 / 10000.0078.75 / 1250.003600.00 / 40000.00✓default
m7g.xlarge 11250.00 / 10000.00156.25 / 1250.006000.00 / 40000.00✓default
m7g.2xlarge 12500.00 / 10000.00312.50 / 1250.0012000.00 / 40000.00✓default
m7g.4xlarge 15000.00 / 10000.00625.00 / 1250.0020000.00 / 40000.00✓default
m7g.8xlarge10000.001250.0040000.00✓default
m7g.12xlarge15000.001875.0060000.00✓default
m7g.16xlarge20000.002500.0080000.00✓default
m7g.metal20000.002500.0080000.00✓default
M7gd
m7gd.medium 1315.00 / 10000.0039.38 / 1250.002500.00 / 40000.00✓default
m7gd.large 1630.00 / 10000.0078.75 / 1250.003600.00 / 40000.00✓default
m7gd.xlarge 11250.00 / 10000.00156.25 / 1250.006000.00 / 40000.00✓default
m7gd.2xlarge 12500.00 / 10000.00312.50 / 1250.0012000.00 / 40000.00✓default
m7gd.4xlarge 15000.00 / 10000.00625.00 / 1250.0020000.00 / 40000.00✓default
m7gd.8xlarge10000.001250.0040000.00✓default
m7gd.12xlarge15000.001875.0060000.00✓default
m7gd.16xlarge20000.002500.0080000.00✓default
m7gd.metal20000.002500.0080000.00✓default
M7i
m7i.large 1650.00 / 10000.0081.25 / 1250.003600.00 / 40000.00✓default
m7i.xlarge 11250.00 / 10000.00156.25 / 1250.006000.00 / 40000.00✓default
m7i.2xlarge 12500.00 / 10000.00312.50 / 1250.0012000.00 / 40000.00✓default
m7i.4xlarge 15000.00 / 10000.00625.00 / 1250.0020000.00 / 40000.00✓default
m7i.8xlarge10000.001250.0040000.00✓default
m7i.12xlarge15000.001875.0060000.00✓default
m7i.16xlarge20000.002500.0080000.00✓default
m7i.24xlarge30000.003750.00120000.00✓default
m7i.48xlarge40000.005000.00240000.00✓default
m7i.metal-24xl30000.003750.00120000.00✓default
m7i.metal-48xl40000.005000.00240000.00✓default
M7i-flex
m7i-flex.large 1312.00 / 10000.0039.06 / 1250.002500.00 / 40000.00✓default
m7i-flex.xlarge 1625.00 / 10000.0078.12 / 1250.003600.00 / 40000.00✓default
m7i-flex.2xlarge 11250.00 / 10000.00156.25 / 1250.006000.00 / 40000.00✓default
m7i-flex.4xlarge 12500.00 / 10000.00312.50 / 1250.0012000.00 / 40000.00✓default
m7i-flex.8xlarge 15000.00 / 10000.00625.00 / 1250.0020000.00 / 40000.00✓default
Mac1
mac1.metal14000.001750.0080000.00✓default
Mac2
mac2.metal10000.001250.0055000.00✓default
Mac2-m2
mac2-m2.metal8000.001000.0055000.00✓default
Mac2-m2pro
mac2-m2pro.metal8000.001000.0055000.00✓default
T2
T3
t3.nano 143.00 / 2085.005.38 / 260.62250.00 / 11800.00✓default
t3.micro 187.00 / 2085.0010.88 / 260.62500.00 / 11800.00✓default
t3.small 1174.00 / 2085.0021.75 / 260.621000.00 / 11800.00✓default
t3.medium 1347.00 / 2085.0043.38 / 260.622000.00 / 11800.00✓default
t3.large 1695.00 / 2780.0086.88 / 347.504000.00 / 15700.00✓default
t3.xlarge 1695.00 / 2780.0086.88 / 347.504000.00 / 15700.00✓default
t3.2xlarge 1695.00 / 2780.0086.88 / 347.504000.00 / 15700.00✓default
T3a
t3a.nano 145.00 / 2085.005.62 / 260.62250.00 / 11800.00✓default
t3a.micro 190.00 / 2085.0011.25 / 260.62500.00 / 11800.00✓default
t3a.small 1175.00 / 2085.0021.88 / 260.621000.00 / 11800.00✓default
t3a.medium 1350.00 / 2085.0043.75 / 260.622000.00 / 11800.00✓default
t3a.large 1695.00 / 2780.0086.88 / 347.504000.00 / 15700.00✓default
t3a.xlarge 1695.00 / 2780.0086.88 / 347.504000.00 / 15700.00✓default
t3a.2xlarge 1695.00 / 2780.0086.88 / 347.504000.00 / 15700.00✓default
T4g
t4g.nano 143.00 / 2085.005.38 / 260.62250.00 / 11800.00✓default
t4g.micro 187.00 / 2085.0010.88 / 260.62500.00 / 11800.00✓default
t4g.small 1174.00 / 2085.0021.75 / 260.621000.00 / 11800.00✓default
t4g.medium 1347.00 / 2085.0043.38 / 260.622000.00 / 11800.00✓default
t4g.large 1695.00 / 2780.0086.88 / 347.504000.00 / 15700.00✓default
t4g.xlarge 1695.00 / 2780.0086.88 / 347.504000.00 / 15700.00✓default
t4g.2xlarge 1695.00 / 2780.0086.88 / 347.504000.00 / 15700.00✓default
+
Note

1 These instances can support maximum performance for 30 minutes at + least once every 24 hours, after which they revert to their baseline performance. + Other instances can sustain the maximum performance indefinitely. If your workload requires + sustained maximum performance for longer than 30 minutes, use one of these instances.

2 default indicates that instances are enabled + for EBS optimization by default. supported indicates that instances can optionally + be enabled for EBS optimization For more information, see Amazon EBS–optimized instances.

+ +

Instance store specifications

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Instance typeInstance store volumesInstance store typeRead / Write performance (IOPS)Needs initialization 1TRIM support 2
M5ad
m5ad.large1 x 75 GBNVMe SSD30,000 / 15,000 ✓
m5ad.xlarge1 x 150 GBNVMe SSD59,000 / 29,000 ✓
m5ad.2xlarge1 x 300 GBNVMe SSD117,000 / 57,000 ✓
m5ad.4xlarge2 x 300 GBNVMe SSD234,000 / 114,000 ✓
m5ad.8xlarge2 x 600 GBNVMe SSD466,666 / 233,334 ✓
m5ad.12xlarge2 x 900 GBNVMe SSD700,000 / 340,000 ✓
m5ad.16xlarge4 x 600 GBNVMe SSD933,332 / 466,668 ✓
m5ad.24xlarge4 x 900 GBNVMe SSD1,400,000 / 680,000 ✓
M5d
m5d.large1 x 75 GBNVMe SSD30,000 / 15,000 ✓
m5d.xlarge1 x 150 GBNVMe SSD59,000 / 29,000 ✓
m5d.2xlarge1 x 300 GBNVMe SSD117,000 / 57,000 ✓
m5d.4xlarge2 x 300 GBNVMe SSD234,000 / 114,000 ✓
m5d.8xlarge2 x 600 GBNVMe SSD466,666 / 233,334 ✓
m5d.12xlarge2 x 900 GBNVMe SSD700,000 / 340,000 ✓
m5d.16xlarge4 x 600 GBNVMe SSD933,332 / 466,668 ✓
m5d.24xlarge4 x 900 GBNVMe SSD1,400,000 / 680,000 ✓
m5d.metal4 x 900 GBNVMe SSD1,400,000 / 680,000 ✓
M5dn
m5dn.large1 x 75 GBNVMe SSD29,000 / 14,500 ✓
m5dn.xlarge1 x 150 GBNVMe SSD58,000 / 29,000 ✓
m5dn.2xlarge1 x 300 GBNVMe SSD116,000 / 58,000 ✓
m5dn.4xlarge2 x 300 GBNVMe SSD232,000 / 116,000 ✓
m5dn.8xlarge2 x 600 GBNVMe SSD464,000 / 232,000 ✓
m5dn.12xlarge2 x 900 GBNVMe SSD700,000 / 350,000 ✓
m5dn.16xlarge4 x 600 GBNVMe SSD930,000 / 465,000 ✓
m5dn.24xlarge4 x 900 GBNVMe SSD1,400,000 / 700,000 ✓
m5dn.metal4 x 900 GBNVMe SSD1,400,000 / 700,000 ✓
M6gd
m6gd.medium1 x 59 GBNVMe SSD13,438 / 5,625 ✓
m6gd.large1 x 118 GBNVMe SSD26,875 / 11,250 ✓
m6gd.xlarge1 x 237 GBNVMe SSD53,750 / 22,500 ✓
m6gd.2xlarge1 x 474 GBNVMe SSD107,500 / 45,000 ✓
m6gd.4xlarge1 x 950 GBNVMe SSD215,000 / 90,000 ✓
m6gd.8xlarge1 x 1900 GBNVMe SSD430,000 / 180,000 ✓
m6gd.12xlarge2 x 1425 GBNVMe SSD645,000 / 270,000 ✓
m6gd.16xlarge2 x 1900 GBNVMe SSD860,000 / 360,000 ✓
m6gd.metal2 x 1900 GBNVMe SSD860,000 / 360,000 ✓
M6id
m6id.large1 x 118 GBNVMe SSD33,542 / 16,771 ✓
m6id.xlarge1 x 237 GBNVMe SSD67,083 / 33,542 ✓
m6id.2xlarge1 x 474 GBNVMe SSD134,167 / 67,084 ✓
m6id.4xlarge1 x 950 GBNVMe SSD268,333 / 134,167 ✓
m6id.8xlarge1 x 1900 GBNVMe SSD536,666 / 268,334 ✓
m6id.12xlarge2 x 1425 GBNVMe SSD804,998 / 402,500 ✓
m6id.16xlarge2 x 1900 GBNVMe SSD1,073,332 / 536,668 ✓
m6id.24xlarge4 x 1425 GBNVMe SSD1,609,996 / 805,000 ✓
m6id.32xlarge4 x 1900 GBNVMe SSD2,146,664 / 1,073,336 ✓
m6id.metal4 x 1900 GBNVMe SSD2,146,664 / 1,073,336 ✓
M6idn
m6idn.large1 x 118 GBNVMe SSD33,542 / 16,771 ✓
m6idn.xlarge1 x 237 GBNVMe SSD67,083 / 33,542 ✓
m6idn.2xlarge1 x 474 GBNVMe SSD134,167 / 67,084 ✓
m6idn.4xlarge1 x 950 GBNVMe SSD268,333 / 134,167 ✓
m6idn.8xlarge1 x 1900 GBNVMe SSD536,666 / 268,334 ✓
m6idn.12xlarge2 x 1425 GBNVMe SSD804,998 / 402,500 ✓
m6idn.16xlarge2 x 1900 GBNVMe SSD1,073,332 / 536,668 ✓
m6idn.24xlarge4 x 1425 GBNVMe SSD1,609,996 / 805,000 ✓
m6idn.32xlarge4 x 1900 GBNVMe SSD2,146,664 / 1,073,336 ✓
m6idn.metal4 x 1900 GBNVMe SSD2,146,664 / 1,073,336 ✓
M7gd
m7gd.medium1 x 59 GBNVMe SSD16,771 / 8,385 ✓
m7gd.large1 x 118 GBNVMe SSD33,542 / 16,771 ✓
m7gd.xlarge1 x 237 GBNVMe SSD67,083 / 33,542 ✓
m7gd.2xlarge1 x 474 GBNVMe SSD134,167 / 67,084 ✓
m7gd.4xlarge1 x 950 GBNVMe SSD268,333 / 134,167 ✓
m7gd.8xlarge1 x 1900 GBNVMe SSD536,666 / 268,334 ✓
m7gd.12xlarge2 x 1425 GBNVMe SSD804,998 / 402,500 ✓
m7gd.16xlarge2 x 1900 GBNVMe SSD1,073,332 / 536,668 ✓
m7gd.metal2 x 1900 GBNVMe SSD1,073,332 / 536,668 ✓
+

1 Volumes attached to certain instances suffer a first-write + penalty unless initialized. For more information, see Optimize disk performance for + instance store volumes.

+

2 For more information, see Instance + store volume TRIM support.

+ +

Security specifications

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Instance typeEBS encryptionInstance store encryptionEncryption in transitAMD SEV-SNPNitroTPMNitro Enclaves
M5
m5.large✓Instance store not supported✗✗✓✗
m5.xlarge✓Instance store not supported✗✗✓✓
m5.2xlarge✓Instance store not supported✗✗✓✓
m5.4xlarge✓Instance store not supported✗✗✓✓
m5.8xlarge✓Instance store not supported✗✗✓✓
m5.12xlarge✓Instance store not supported✗✗✓✓
m5.16xlarge✓Instance store not supported✗✗✓✓
m5.24xlarge✓Instance store not supported✗✗✓✓
m5.metal✓Instance store not supported✗✗✗✗
M5a
m5a.large✓Instance store not supported✗✗✓✗
m5a.xlarge✓Instance store not supported✗✗✓✓
m5a.2xlarge✓Instance store not supported✗✗✓✓
m5a.4xlarge✓Instance store not supported✗✗✓✓
m5a.8xlarge✓Instance store not supported✗✗✓✓
m5a.12xlarge✓Instance store not supported✗✗✓✓
m5a.16xlarge✓Instance store not supported✗✗✓✓
m5a.24xlarge✓Instance store not supported✗✗✓✓
M5ad
m5ad.large✓✓✗✗✓✗
m5ad.xlarge✓✓✗✗✓✓
m5ad.2xlarge✓✓✗✗✓✓
m5ad.4xlarge✓✓✗✗✓✓
m5ad.8xlarge✓✓✗✗✓✓
m5ad.12xlarge✓✓✗✗✓✓
m5ad.16xlarge✓✓✗✗✓✓
m5ad.24xlarge✓✓✗✗✓✓
M5d
m5d.large✓✓✗✗✓✗
m5d.xlarge✓✓✗✗✓✓
m5d.2xlarge✓✓✗✗✓✓
m5d.4xlarge✓✓✗✗✓✓
m5d.8xlarge✓✓✗✗✓✓
m5d.12xlarge✓✓✗✗✓✓
m5d.16xlarge✓✓✗✗✓✓
m5d.24xlarge✓✓✗✗✓✓
m5d.metal✓✓✗✗✗✗
M5dn
m5dn.large✓✓✓✗✓✗
m5dn.xlarge✓✓✓✗✓✓
m5dn.2xlarge✓✓✓✗✓✓
m5dn.4xlarge✓✓✓✗✓✓
m5dn.8xlarge✓✓✓✗✓✓
m5dn.12xlarge✓✓✓✗✓✓
m5dn.16xlarge✓✓✓✗✓✓
m5dn.24xlarge✓✓✓✗✓✓
m5dn.metal✓✓✓✗✗✗
M5n
m5n.large✓Instance store not supported✓✗✓✗
m5n.xlarge✓Instance store not supported✓✗✓✓
m5n.2xlarge✓Instance store not supported✓✗✓✓
m5n.4xlarge✓Instance store not supported✓✗✓✓
m5n.8xlarge✓Instance store not supported✓✗✓✓
m5n.12xlarge✓Instance store not supported✓✗✓✓
m5n.16xlarge✓Instance store not supported✓✗✓✓
m5n.24xlarge✓Instance store not supported✓✗✓✓
m5n.metal✓Instance store not supported✓✗✗✗
M5zn
m5zn.large✓Instance store not supported✓✗✓✗
m5zn.xlarge✓Instance store not supported✓✗✓✓
m5zn.2xlarge✓Instance store not supported✓✗✓✓
m5zn.3xlarge✓Instance store not supported✓✗✓✓
m5zn.6xlarge✓Instance store not supported✓✗✓✓
m5zn.12xlarge✓Instance store not supported✓✗✓✓
m5zn.metal✓Instance store not supported✓✗✗✗
M6a
m6a.large✓Instance store not supported✓✓✓✗
m6a.xlarge✓Instance store not supported✓✓✓✓
m6a.2xlarge✓Instance store not supported✓✓✓✓
m6a.4xlarge✓Instance store not supported✓✓✓✓
m6a.8xlarge✓Instance store not supported✓✓✓✓
m6a.12xlarge✓Instance store not supported✓✗✓✓
m6a.16xlarge✓Instance store not supported✓✗✓✓
m6a.24xlarge✓Instance store not supported✓✗✓✓
m6a.32xlarge✓Instance store not supported✓✗✓✓
m6a.48xlarge✓Instance store not supported✓✗✓✓
m6a.metal✓Instance store not supported✓✗✗✗
M6g
m6g.medium✓Instance store not supported✗✗✗✗
m6g.large✓Instance store not supported✗✗✗✓
m6g.xlarge✓Instance store not supported✗✗✗✓
m6g.2xlarge✓Instance store not supported✗✗✗✓
m6g.4xlarge✓Instance store not supported✗✗✗✓
m6g.8xlarge✓Instance store not supported✗✗✗✓
m6g.12xlarge✓Instance store not supported✗✗✗✓
m6g.16xlarge✓Instance store not supported✗✗✗✓
m6g.metal✓Instance store not supported✗✗✗✗
M6gd
m6gd.medium✓✓✗✗✗✗
m6gd.large✓✓✗✗✗✓
m6gd.xlarge✓✓✗✗✗✓
m6gd.2xlarge✓✓✗✗✗✓
m6gd.4xlarge✓✓✗✗✗✓
m6gd.8xlarge✓✓✗✗✗✓
m6gd.12xlarge✓✓✗✗✗✓
m6gd.16xlarge✓✓✗✗✗✓
m6gd.metal✓✓✗✗✗✗
M6i
m6i.large✓Instance store not supported✓✗✓✗
m6i.xlarge✓Instance store not supported✓✗✓✓
m6i.2xlarge✓Instance store not supported✓✗✓✓
m6i.4xlarge✓Instance store not supported✓✗✓✓
m6i.8xlarge✓Instance store not supported✓✗✓✓
m6i.12xlarge✓Instance store not supported✓✗✓✓
m6i.16xlarge✓Instance store not supported✓✗✓✓
m6i.24xlarge✓Instance store not supported✓✗✓✓
m6i.32xlarge✓Instance store not supported✓✗✓✓
m6i.metal✓Instance store not supported✓✗✗✗
M6id
m6id.large✓✓✓✗✓✗
m6id.xlarge✓✓✓✗✓✓
m6id.2xlarge✓✓✓✗✓✓
m6id.4xlarge✓✓✓✗✓✓
m6id.8xlarge✓✓✓✗✓✓
m6id.12xlarge✓✓✓✗✓✓
m6id.16xlarge✓✓✓✗✓✓
m6id.24xlarge✓✓✓✗✓✓
m6id.32xlarge✓✓✓✗✓✓
m6id.metal✓✓✓✗✗✗
M6idn
m6idn.large✓✓✓✗✓✗
m6idn.xlarge✓✓✓✗✓✓
m6idn.2xlarge✓✓✓✗✓✓
m6idn.4xlarge✓✓✓✗✓✓
m6idn.8xlarge✓✓✓✗✓✓
m6idn.12xlarge✓✓✓✗✓✓
m6idn.16xlarge✓✓✓✗✓✓
m6idn.24xlarge✓✓✓✗✓✓
m6idn.32xlarge✓✓✓✗✓✓
m6idn.metal✓✓✓✗✗✗
M6in
m6in.large✓Instance store not supported✓✗✓✗
m6in.xlarge✓Instance store not supported✓✗✓✓
m6in.2xlarge✓Instance store not supported✓✗✓✓
m6in.4xlarge✓Instance store not supported✓✗✓✓
m6in.8xlarge✓Instance store not supported✓✗✓✓
m6in.12xlarge✓Instance store not supported✓✗✓✓
m6in.16xlarge✓Instance store not supported✓✗✓✓
m6in.24xlarge✓Instance store not supported✓✗✓✓
m6in.32xlarge✓Instance store not supported✓✗✓✓
m6in.metal✓Instance store not supported✓✗✗✗
M7a
m7a.medium✓Instance store not supported✓✗✓✗
m7a.large✓Instance store not supported✓✗✓✗
m7a.xlarge✓Instance store not supported✓✗✓✗
m7a.2xlarge✓Instance store not supported✓✗✓✗
m7a.4xlarge✓Instance store not supported✓✗✓✗
m7a.8xlarge✓Instance store not supported✓✗✓✗
m7a.12xlarge✓Instance store not supported✓✗✓✗
m7a.16xlarge✓Instance store not supported✓✗✓✗
m7a.24xlarge✓Instance store not supported✓✗✓✗
m7a.32xlarge✓Instance store not supported✓✗✓✗
m7a.48xlarge✓Instance store not supported✓✗✓✗
m7a.metal-48xl✓Instance store not supported✓✗✗✗
M7g
m7g.medium✓Instance store not supported✓✗✗✗
m7g.large✓Instance store not supported✓✗✗✗
m7g.xlarge✓Instance store not supported✓✗✗✗
m7g.2xlarge✓Instance store not supported✓✗✗✗
m7g.4xlarge✓Instance store not supported✓✗✗✗
m7g.8xlarge✓Instance store not supported✓✗✗✗
m7g.12xlarge✓Instance store not supported✓✗✗✗
m7g.16xlarge✓Instance store not supported✓✗✗✗
m7g.metal✓Instance store not supported✓✗✗✗
M7gd
m7gd.medium✓✓✓✗✗✗
m7gd.large✓✓✓✗✗✗
m7gd.xlarge✓✓✓✗✗✗
m7gd.2xlarge✓✓✓✗✗✗
m7gd.4xlarge✓✓✓✗✗✗
m7gd.8xlarge✓✓✓✗✗✗
m7gd.12xlarge✓✓✓✗✗✗
m7gd.16xlarge✓✓✓✗✗✗
m7gd.metal✓✓✓✗✗✗
M7i
m7i.large✓Instance store not supported✓✗✓✗
m7i.xlarge✓Instance store not supported✓✗✓✗
m7i.2xlarge✓Instance store not supported✓✗✓✗
m7i.4xlarge✓Instance store not supported✓✗✓✗
m7i.8xlarge✓Instance store not supported✓✗✓✗
m7i.12xlarge✓Instance store not supported✓✗✓✗
m7i.16xlarge✓Instance store not supported✓✗✓✗
m7i.24xlarge✓Instance store not supported✓✗✓✗
m7i.48xlarge✓Instance store not supported✓✗✓✗
m7i.metal-24xl✓Instance store not supported✓✗✗✗
m7i.metal-48xl✓Instance store not supported✓✗✗✗
M7i-flex
m7i-flex.large✓Instance store not supported✓✗✓✗
m7i-flex.xlarge✓Instance store not supported✓✗✓✗
m7i-flex.2xlarge✓Instance store not supported✓✗✓✗
m7i-flex.4xlarge✓Instance store not supported✓✗✓✗
m7i-flex.8xlarge✓Instance store not supported✓✗✓✗
Mac1
mac1.metal✓Instance store not supported✗✗✗✗
Mac2
mac2.metal✓Instance store not supported✗✗✗✗
Mac2-m2
mac2-m2.metal✓Instance store not supported✗✗✗✗
Mac2-m2pro
mac2-m2pro.metal✓Instance store not supported✗✗✗✗
T2
t2.nano✓Instance store not supported✗✗✗✗
t2.micro✓Instance store not supported✗✗✗✗
t2.small✓Instance store not supported✗✗✗✗
t2.medium✓Instance store not supported✗✗✗✗
t2.large✓Instance store not supported✗✗✗✗
t2.xlarge✓Instance store not supported✗✗✗✗
t2.2xlarge✓Instance store not supported✗✗✗✗
T3
t3.nano✓Instance store not supported✗✗✓✗
t3.micro✓Instance store not supported✗✗✓✗
t3.small✓Instance store not supported✗✗✓✗
t3.medium✓Instance store not supported✗✗✓✗
t3.large✓Instance store not supported✗✗✓✗
t3.xlarge✓Instance store not supported✗✗✓✗
t3.2xlarge✓Instance store not supported✗✗✓✗
T3a
t3a.nano✓Instance store not supported✗✗✓✗
t3a.micro✓Instance store not supported✗✗✓✗
t3a.small✓Instance store not supported✗✗✓✗
t3a.medium✓Instance store not supported✗✗✓✗
t3a.large✓Instance store not supported✗✗✓✗
t3a.xlarge✓Instance store not supported✗✗✓✗
t3a.2xlarge✓Instance store not supported✗✗✓✗
T4g
t4g.nano✓Instance store not supported✗✗✗✗
t4g.micro✓Instance store not supported✗✗✗✗
t4g.small✓Instance store not supported✗✗✗✗
t4g.medium✓Instance store not supported✗✗✗✗
t4g.large✓Instance store not supported✗✗✗✗
t4g.xlarge✓Instance store not supported✗✗✗✗
t4g.2xlarge✓Instance store not supported✗✗✗✗
+
\ No newline at end of file diff --git a/hack/code/bandwidth_gen/main.go b/hack/code/bandwidth_gen/main.go index 3bbdc2e132fd..7ec947dbdd8a 100644 --- a/hack/code/bandwidth_gen/main.go +++ b/hack/code/bandwidth_gen/main.go @@ -32,12 +32,13 @@ import ( ) var uriSelectors = map[string]string{ - "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/general-purpose-instances.html": "#general-purpose-network-performance", - "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/compute-optimized-instances.html": "#compute-network-performance", - "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/memory-optimized-instances.html": "#memory-network-perf", - "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/storage-optimized-instances.html": "#storage-network-performance", - "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/accelerated-computing-instances.html": "#gpu-network-performance", - "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/high-performance-computing-instances.html": "#hpc-network-performance", + "https://docs.aws.amazon.com/ec2/latest/instancetypes/gp.html": "#gp_network", + "https://docs.aws.amazon.com/ec2/latest/instancetypes/co.html": "#co_network", + "https://docs.aws.amazon.com/ec2/latest/instancetypes/mo.html": "#mo_network", + "https://docs.aws.amazon.com/ec2/latest/instancetypes/so.html": "#so_network", + "https://docs.aws.amazon.com/ec2/latest/instancetypes/ac.html": "#ac_network", + "https://docs.aws.amazon.com/ec2/latest/instancetypes/hpc.html": "#hpc_network", + "https://docs.aws.amazon.com/ec2/latest/instancetypes/pg.html": "#pg_network", } const fileFormat = ` @@ -62,6 +63,7 @@ func main() { } bandwidth := map[string]int64{} + vagueBandwidth := map[string]string{} for uri, selector := range uriSelectors { func() { @@ -70,16 +72,18 @@ func main() { doc := lo.Must(goquery.NewDocumentFromReader(response.Body)) - // grab two tables that contain the network performance values - // first table will contain all the instance type and bandwidth data - // some rows will will have vague describe such as "Very Low", "Low", "Low to Moderate", etc. - // These instance types will can be found on the second table with absolute values in Gbps - // If the instance type is skipped on the first table it will be grabbed on the second table + // grab the table that contains the network performance values. Some instance types will have vague + // description for bandwidth such as "Very Low", "Low", "Low to Moderate", etc. These instance types + // will be ignored since we don't know the exact bandwidth for these instance types for _, row := range doc.Find(selector).NextAllFiltered(".table-container").Eq(0).Find("tbody").Find("tr").Nodes { - instanceTypeData := row.FirstChild.NextSibling.FirstChild.FirstChild.Data + instanceTypeData := strings.TrimSpace(row.FirstChild.NextSibling.FirstChild.Data) + if !strings.ContainsAny(instanceTypeData, ".") { + continue + } bandwidthData := row.FirstChild.NextSibling.NextSibling.NextSibling.FirstChild.Data // exclude all rows that contain any of the following strings if containsAny(bandwidthData, "Low", "Moderate", "High", "Up to") { + vagueBandwidth[instanceTypeData] = bandwidthData continue } bandwidthSlice := strings.Split(bandwidthData, " ") @@ -92,30 +96,9 @@ func main() { bandwidth[instanceTypeData] = int64(lo.Must(strconv.ParseFloat(bandwidthSlice[0], 64)) * 1000) } } - - // Collect instance types bandwidth data from the baseline/bandwidth table underneath the standard table - // The HPC network performance doc is laid out differently than the other docs. There is no table underneath - // the standard table that contains information for network performance with baseline and burst bandwidth. - if selector != "#hpc-network-performance" { - for _, row := range doc.Find(selector).NextAllFiltered(".table-container").Eq(1).Find("tbody").Find("tr").Nodes { - instanceTypeData := row.FirstChild.NextSibling.FirstChild.FirstChild.Data - bandwidthData := row.FirstChild.NextSibling.NextSibling.NextSibling.FirstChild.Data - bandwidth[instanceTypeData] = int64(lo.Must(strconv.ParseFloat(bandwidthData, 64)) * 1000) - } - } }() } - if err := os.Setenv("AWS_SDK_LOAD_CONFIG", "true"); err != nil { - log.Fatalf("setting AWS_SDK_LOAD_CONFIG, %s", err) - } - if err := os.Setenv("AWS_REGION", "us-east-1"); err != nil { - log.Fatalf("setting AWS_REGION, %s", err) - } - sess := session.Must(session.NewSession()) - ec2api := ec2.New(sess) - instanceTypesOutput := lo.Must(ec2api.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{})) - allInstanceTypes := lo.Map(instanceTypesOutput.InstanceTypes, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType }) - + allInstanceTypes := getAllInstanceTypes() instanceTypes := lo.Keys(bandwidth) // 2d sort for readability sort.Strings(allInstanceTypes) @@ -127,6 +110,10 @@ func main() { // Generate body var body string for _, instanceType := range lo.Without(allInstanceTypes, instanceTypes...) { + if lo.Contains(lo.Keys(vagueBandwidth), instanceType) { + body += fmt.Sprintf("// %s has vague bandwidth information, bandwidth is %s\n", instanceType, vagueBandwidth[instanceType]) + continue + } body += fmt.Sprintf("// %s is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html\n", instanceType) } for _, instanceType := range instanceTypes { @@ -150,3 +137,29 @@ func containsAny(value string, excludedSubstrings ...string) bool { } return false } + +func getAllInstanceTypes() []string { + if err := os.Setenv("AWS_SDK_LOAD_CONFIG", "true"); err != nil { + log.Fatalf("setting AWS_SDK_LOAD_CONFIG, %s", err) + } + if err := os.Setenv("AWS_REGION", "us-east-1"); err != nil { + log.Fatalf("setting AWS_REGION, %s", err) + } + sess := session.Must(session.NewSession()) + ec2api := ec2.New(sess) + var allInstanceTypes []string + + params := &ec2.DescribeInstanceTypesInput{} + // Retrieve the instance types in a loop using NextToken + for { + result := lo.Must(ec2api.DescribeInstanceTypes(params)) + allInstanceTypes = append(allInstanceTypes, lo.Map(result.InstanceTypes, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType })...) + // Check if they are any instances left + if result.NextToken != nil { + params.NextToken = result.NextToken + } else { + break + } + } + return allInstanceTypes +} diff --git a/hack/code/instancetype_testdata_gen/main.go b/hack/code/instancetype_testdata_gen/main.go index 9069cb63cd60..e0df1b16163d 100644 --- a/hack/code/instancetype_testdata_gen/main.go +++ b/hack/code/instancetype_testdata_gen/main.go @@ -130,6 +130,23 @@ func getInstanceTypeInfo(info *ec2.InstanceTypeInfo) string { fmt.Fprintf(src, "SizeInMiB: aws.Int64(%d),\n", lo.FromPtr(info.MemoryInfo.SizeInMiB)) fmt.Fprintf(src, "},\n") + if info.EbsInfo != nil { + fmt.Fprintf(src, "EbsInfo: &ec2.EbsInfo{\n") + if info.EbsInfo.EbsOptimizedInfo != nil { + fmt.Fprintf(src, "EbsOptimizedInfo: &ec2.EbsOptimizedInfo{\n") + fmt.Fprintf(src, "BaselineBandwidthInMbps: aws.Int64(%d),\n", lo.FromPtr(info.EbsInfo.EbsOptimizedInfo.BaselineBandwidthInMbps)) + fmt.Fprintf(src, "BaselineIops: aws.Int64(%d),\n", lo.FromPtr(info.EbsInfo.EbsOptimizedInfo.BaselineIops)) + fmt.Fprintf(src, "BaselineThroughputInMBps: aws.Float64(%.2f),\n", lo.FromPtr(info.EbsInfo.EbsOptimizedInfo.BaselineThroughputInMBps)) + fmt.Fprintf(src, "MaximumBandwidthInMbps: aws.Int64(%d),\n", lo.FromPtr(info.EbsInfo.EbsOptimizedInfo.MaximumBandwidthInMbps)) + fmt.Fprintf(src, "MaximumIops: aws.Int64(%d),\n", lo.FromPtr(info.EbsInfo.EbsOptimizedInfo.MaximumIops)) + fmt.Fprintf(src, "MaximumThroughputInMBps: aws.Float64(%.2f),\n", lo.FromPtr(info.EbsInfo.EbsOptimizedInfo.MaximumThroughputInMBps)) + fmt.Fprintf(src, "},\n") + } + fmt.Fprintf(src, "EbsOptimizedSupport: aws.String(\"%s\"),\n", lo.FromPtr(info.EbsInfo.EbsOptimizedSupport)) + fmt.Fprintf(src, "EncryptionSupport: aws.String(\"%s\"),\n", lo.FromPtr(info.EbsInfo.EncryptionSupport)) + fmt.Fprintf(src, "NvmeSupport: aws.String(\"%s\"),\n", lo.FromPtr(info.EbsInfo.NvmeSupport)) + fmt.Fprintf(src, "},\n") + } if info.InferenceAcceleratorInfo != nil { fmt.Fprintf(src, "InferenceAcceleratorInfo: &ec2.InferenceAcceleratorInfo{\n") fmt.Fprintf(src, "Accelerators: []*ec2.InferenceDeviceInfo{\n") diff --git a/hack/code/prices_gen/main.go b/hack/code/prices_gen/main.go index 7b73fdfc8564..837eeaedb360 100644 --- a/hack/code/prices_gen/main.go +++ b/hack/code/prices_gen/main.go @@ -31,10 +31,8 @@ import ( "github.com/aws/aws-sdk-go/aws/session" ec22 "github.com/aws/aws-sdk-go/service/ec2" "github.com/samber/lo" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/pricing" + controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/pricing" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" "github.com/aws/karpenter-provider-aws/pkg/test" @@ -110,7 +108,7 @@ func main() { log.Println("fetching for", region) pricingProvider := pricing.NewDefaultProvider(ctx, pricing.NewAPI(sess, region), ec2, region) controller := controllerspricing.NewController(pricingProvider) - _, err := controller.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{}}) + _, err := controller.Reconcile(ctx) if err != nil { log.Fatalf("failed to initialize pricing provider %s", err) } diff --git a/hack/codegen.sh b/hack/codegen.sh index ec603b72ff7c..f148e0ca1ca7 100755 --- a/hack/codegen.sh +++ b/hack/codegen.sh @@ -46,7 +46,7 @@ instanceTypeTestData() { GENERATED_FILE="pkg/fake/zz_generated.describe_instance_types.go" go run hack/code/instancetype_testdata_gen/main.go --out-file ${GENERATED_FILE} \ - --instance-types t3.large,m5.large,m5.xlarge,p3.8xlarge,g4dn.8xlarge,c6g.large,inf1.2xlarge,inf1.6xlarge,trn1.2xlarge,m5.metal,dl1.24xlarge,m6idn.32xlarge,t4g.small,t4g.xlarge,t4g.medium + --instance-types t3.large,m5.large,m5.xlarge,p3.8xlarge,g4dn.8xlarge,c6g.large,inf1.2xlarge,inf1.6xlarge,trn1.2xlarge,m5.metal,dl1.24xlarge,m6idn.32xlarge,t4g.small,t4g.xlarge,t4g.medium,g4ad.16xlarge checkForUpdates "${GENERATED_FILE}" } diff --git a/hack/docgen.sh b/hack/docgen.sh index 037114698363..e868e2607214 100755 --- a/hack/docgen.sh +++ b/hack/docgen.sh @@ -4,13 +4,16 @@ set -euo pipefail compatibilitymatrix() { # versionCount is the number of K8s versions to display in the compatibility matrix versionCount=7 - go run hack/docs/version_compatibility.go hack/docs/compatibility-karpenter.yaml "$(git describe --exact-match --tags || echo "no tag")" - go run hack/docs/compatibilitymetrix_gen_docs.go website/content/en/preview/upgrading/compatibility.md hack/docs/compatibility-karpenter.yaml $versionCount + go run hack/docs/version_compatibility_gen/main.go hack/docs/compatibilitymatrix_gen/compatibility.yaml "$(git describe --exact-match --tags || echo "no tag")" + go run hack/docs/compatibilitymatrix_gen/main.go website/content/en/preview/upgrading/compatibility.md hack/docs/compatibilitymatrix_gen/compatibility.yaml $versionCount } +CONTROLLER_RUNTIME_DIR=$(go list -m -f '{{ .Dir }}' sigs.k8s.io/controller-runtime) +AWS_SDK_GO_PROMETHEUS_DIR=$(go list -m -f '{{ .Dir }}' github.com/jonathan-innis/aws-sdk-go-prometheus) +OPERATORPKG_DIR=$(go list -m -f '{{ .Dir }}' github.com/awslabs/operatorpkg) compatibilitymatrix -go run hack/docs/metrics_gen_docs.go pkg/ ${KARPENTER_CORE_DIR}/pkg website/content/en/preview/reference/metrics.md -go run hack/docs/instancetypes_gen_docs.go website/content/en/preview/reference/instance-types.md -go run hack/docs/configuration_gen_docs.go website/content/en/preview/reference/settings.md +go run hack/docs/metrics_gen/main.go pkg/ "${KARPENTER_CORE_DIR}/pkg" "${CONTROLLER_RUNTIME_DIR}/pkg" "${AWS_SDK_GO_PROMETHEUS_DIR}" "${OPERATORPKG_DIR}" website/content/en/preview/reference/metrics.md +go run hack/docs/instancetypes_gen/main.go website/content/en/preview/reference/instance-types.md +go run hack/docs/configuration_gen/main.go website/content/en/preview/reference/settings.md cd charts/karpenter && helm-docs diff --git a/hack/docs/compatibility-karpenter.yaml b/hack/docs/compatibilitymatrix_gen/compatibility.yaml similarity index 92% rename from hack/docs/compatibility-karpenter.yaml rename to hack/docs/compatibilitymatrix_gen/compatibility.yaml index 445175efcb2b..19440233d155 100644 --- a/hack/docs/compatibility-karpenter.yaml +++ b/hack/docs/compatibilitymatrix_gen/compatibility.yaml @@ -47,4 +47,7 @@ compatibility: maxK8sVersion: 1.29 - appVersion: 0.36.0 minK8sVersion: 1.23 - maxK8sVersion: 1.29 \ No newline at end of file + maxK8sVersion: 1.29 + - appVersion: 0.37.0 + minK8sVersion: 1.23 + maxK8sVersion: 1.30 \ No newline at end of file diff --git a/hack/docs/compatibilitymetrix_gen_docs.go b/hack/docs/compatibilitymatrix_gen/main.go similarity index 100% rename from hack/docs/compatibilitymetrix_gen_docs.go rename to hack/docs/compatibilitymatrix_gen/main.go diff --git a/hack/docs/configuration_gen_docs.go b/hack/docs/configuration_gen/main.go similarity index 100% rename from hack/docs/configuration_gen_docs.go rename to hack/docs/configuration_gen/main.go diff --git a/hack/docs/instancetypes_gen_docs.go b/hack/docs/instancetypes_gen/main.go similarity index 79% rename from hack/docs/instancetypes_gen_docs.go rename to hack/docs/instancetypes_gen/main.go index 68736fb5e823..e5cafcde2803 100644 --- a/hack/docs/instancetypes_gen_docs.go +++ b/hack/docs/instancetypes_gen/main.go @@ -23,8 +23,9 @@ import ( "sort" "strings" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" @@ -32,13 +33,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coreoperator "sigs.k8s.io/karpenter/pkg/operator" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" coretest "sigs.k8s.io/karpenter/pkg/test" - awscloudprovider "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/operator" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/test" @@ -95,10 +96,38 @@ func main() { Manager: &FakeManager{}, KubernetesInterface: kubernetes.NewForConfigOrDie(&rest.Config{}), }) - cp := awscloudprovider.New(op.InstanceTypesProvider, op.InstanceProvider, - op.EventRecorder, op.GetClient(), op.AMIProvider, op.SecurityGroupProvider, op.SubnetProvider) - - instanceTypes, err := cp.GetInstanceTypes(ctx, nil) + if err := op.InstanceTypesProvider.UpdateInstanceTypes(ctx); err != nil { + log.Fatalf("updating instance types, %s", err) + } + if err := op.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx); err != nil { + log.Fatalf("updating instance types offerings, %s", err) + } + // Fake a NodeClass so we can use it to get InstanceTypes + nodeClass := &v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: []v1.AMISelectorTerm{{ + Alias: "al2023@latest", + }}, + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "*": "*", + }, + }, + }, + }, + } + subnets, err := op.SubnetProvider.List(ctx, nodeClass) + if err != nil { + log.Fatalf("listing subnets, %s", err) + } + nodeClass.Status.Subnets = lo.Map(subnets, func(ec2subnet *ec2.Subnet, _ int) v1.Subnet { + return v1.Subnet{ + ID: *ec2subnet.SubnetId, + Zone: *ec2subnet.AvailabilityZone, + } + }) + instanceTypes, err := op.InstanceTypesProvider.List(ctx, &v1.KubeletConfiguration{}, nodeClass) if err != nil { log.Fatalf("listing instance types, %s", err) } @@ -123,8 +152,7 @@ description: > fmt.Fprintln(f, `AWS instance types offer varying resources and can be selected by labels. The values provided below are the resources available with some assumptions and after the instance overhead has been subtracted: - `+"`blockDeviceMappings` are not configured"+` -- `+"`aws-eni-limited-pod-density` is assumed to be `true`"+` -- `+"`amiFamily` is set to the default of `AL2`") +- `+"`amiFamily` is set to `AL2023`") // generate a map of family -> instance types along with some other sorted lists. The sorted lists ensure we // generate consistent docs every run. @@ -145,8 +173,9 @@ below are the resources available with some assumptions and after the instance o sort.Strings(familyNames) // we don't want to show a few labels that will vary amongst regions - delete(labelNameMap, v1.LabelTopologyZone) - delete(labelNameMap, v1beta1.CapacityTypeLabelKey) + delete(labelNameMap, corev1.LabelTopologyZone) + delete(labelNameMap, v1.LabelTopologyZoneID) + delete(labelNameMap, karpv1.CapacityTypeLabelKey) labelNames := lo.Keys(labelNameMap) @@ -183,7 +212,7 @@ below are the resources available with some assumptions and after the instance o if !ok { continue } - if req.Key == v1.LabelTopologyRegion { + if req.Key == corev1.LabelTopologyRegion { continue } if len(req.Values()) == 1 { @@ -194,11 +223,11 @@ below are the resources available with some assumptions and after the instance o fmt.Fprintln(f, " | Resource | Quantity |") fmt.Fprintln(f, " |--|--|") for _, resourceName := range resourceNames { - quantity := minusOverhead[v1.ResourceName(resourceName)] + quantity := minusOverhead[corev1.ResourceName(resourceName)] if quantity.IsZero() { continue } - if v1.ResourceName(resourceName) == v1.ResourceEphemeralStorage { + if corev1.ResourceName(resourceName) == corev1.ResourceEphemeralStorage { i64, _ := quantity.AsInt64() quantity = *resource.NewQuantity(i64, resource.BinarySI) } diff --git a/hack/docs/metrics_gen_docs.go b/hack/docs/metrics_gen/main.go similarity index 59% rename from hack/docs/metrics_gen_docs.go rename to hack/docs/metrics_gen/main.go index ffc7c07ed852..9c3650d45572 100644 --- a/hack/docs/metrics_gen_docs.go +++ b/hack/docs/metrics_gen/main.go @@ -27,6 +27,8 @@ import ( "sort" "strings" + "golang.org/x/exp/slices" + "github.com/samber/lo" "sigs.k8s.io/karpenter/pkg/metrics" @@ -39,6 +41,16 @@ type metricInfo struct { help string } +var ( + stableMetrics = []string{"controller_runtime", "aws_sdk_go", "client_go", "leader_election", "interruption", "cluster_state", "workqueue", "karpenter_build_info", "karpenter_nodepool_usage", "karpenter_nodepool_limit", + "karpenter_nodeclaims_terminated_total", "karpenter_nodeclaims_created_total", "karpenter_nodes_terminated_total", "karpenter_nodes_created_total", "karpenter_pods_startup_duration_seconds", + "karpenter_scheduler_scheduling_duration_seconds", "karpenter_provisioner_scheduling_duration_seconds", "karpenter_nodepool_allowed_disruptions", "karpenter_voluntary_disruption_decisions_total"} + betaMetrics = []string{"status_condition", "cloudprovider", "cloudprovider_batcher", "karpenter_nodeclaims_termination_duration_seconds", "karpenter_nodeclaims_instance_termination_duration_seconds", + "karpenter_nodes_total_pod_requests", "karpenter_nodes_total_pod_limits", "karpenter_nodes_total_daemon_requests", "karpenter_nodes_total_daemon_limits", "karpenter_nodes_termination_duration_seconds", + "karpenter_nodes_system_overhead", "karpenter_nodes_allocatable", "karpenter_pods_state", "karpenter_scheduler_queue_depth", "karpenter_voluntary_disruption_queue_failures_total", + "karpenter_voluntary_disruption_decision_evaluation_duration_seconds", "karpenter_voluntary_disruption_eligible_nodes", "karpenter_voluntary_disruption_consolidation_timeouts_total"} +) + func (i metricInfo) qualifiedName() string { return strings.Join(lo.Compact([]string{i.namespace, i.subsystem, i.name}), "_") } @@ -56,12 +68,27 @@ func main() { packages := getPackages(flag.Arg(i)) allMetrics = append(allMetrics, getMetricsFromPackages(packages...)...) } - // Controller Runtime naming is different in that they don't specify a namespace or subsystem + + // Dedupe metrics + allMetrics = lo.UniqBy(allMetrics, func(m metricInfo) string { + return fmt.Sprintf("%s/%s/%s", m.namespace, m.subsystem, m.name) + }) + + // Drop some metrics + for _, subsystem := range []string{"rest_client", "certwatcher_read", "controller_runtime_webhook"} { + allMetrics = lo.Reject(allMetrics, func(m metricInfo, _ int) bool { + return strings.HasPrefix(m.name, subsystem) + }) + } + + // Controller Runtime and AWS SDK Go for Prometheus naming is different in that they don't specify a namespace or subsystem // Getting the metrics requires special parsing logic - for i := range allMetrics { - if allMetrics[i].subsystem == "" && strings.HasPrefix(allMetrics[i].name, "controller_runtime_") { - allMetrics[i].subsystem = "controller_runtime" - allMetrics[i].name = strings.TrimPrefix(allMetrics[i].name, "controller_runtime_") + for _, subsystem := range []string{"controller_runtime", "aws_sdk_go", "client_go", "leader_election"} { + for i := range allMetrics { + if allMetrics[i].subsystem == "" && strings.HasPrefix(allMetrics[i].name, fmt.Sprintf("%s_", subsystem)) { + allMetrics[i].subsystem = subsystem + allMetrics[i].name = strings.TrimPrefix(allMetrics[i].name, fmt.Sprintf("%s_", subsystem)) + } } } sort.Slice(allMetrics, bySubsystem(allMetrics)) @@ -84,14 +111,18 @@ description: > `) fmt.Fprintf(f, "\n") fmt.Fprintf(f, "Karpenter makes several metrics available in Prometheus format to allow monitoring cluster provisioning status. "+ - "These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8000/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings)\n") + "These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8080/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings)\n") previousSubsystem := "" for _, metric := range allMetrics { if metric.subsystem != previousSubsystem { if metric.subsystem != "" { subsystemTitle := strings.Join(lo.Map(strings.Split(metric.subsystem, "_"), func(s string, _ int) string { - return fmt.Sprintf("%s%s", strings.ToTitle(s[0:1]), s[1:]) + if s == "sdk" || s == "aws" { + return strings.ToUpper(s) + } else { + return fmt.Sprintf("%s%s", strings.ToUpper(s[0:1]), s[1:]) + } }), " ") fmt.Fprintf(f, "## %s Metrics\n", subsystemTitle) fmt.Fprintln(f) @@ -100,6 +131,14 @@ description: > } fmt.Fprintf(f, "### `%s`\n", metric.qualifiedName()) fmt.Fprintf(f, "%s\n", metric.help) + switch { + case slices.Contains(stableMetrics, metric.subsystem) || slices.Contains(stableMetrics, metric.qualifiedName()): + fmt.Fprintf(f, "- Stability Level: %s\n", "STABLE") + case slices.Contains(betaMetrics, metric.subsystem) || slices.Contains(betaMetrics, metric.qualifiedName()): + fmt.Fprintf(f, "- Stability Level: %s\n", "BETA") + default: + fmt.Fprintf(f, "- Stability Level: %s\n", "ALPHA") + } fmt.Fprintln(f) } @@ -162,11 +201,16 @@ func bySubsystem(metrics []metricInfo) func(i int, j int) bool { // Higher ordering comes first. If a value isn't designated here then the subsystem will be given a default of 0. // Metrics without a subsystem come first since there is no designation for the bucket they fall under subSystemSortOrder := map[string]int{ - "": 100, - "nodepool": 10, - "nodeclaim": 9, - "nodes": 8, - "pods": 7, + "": 100, + "nodepool": 10, + "nodeclaims": 9, + "nodes": 8, + "pods": 7, + "status_condition": -1, + "workqueue": -1, + "client_go": -1, + "aws_sdk_go": -1, + "leader_election": -2, } return func(i, j int) bool { @@ -226,7 +270,8 @@ func handleVariableDeclaration(v *ast.GenDecl) []metricInfo { } else { value = v } - + case *ast.BinaryExpr: + value = getBinaryExpr(val) default: log.Fatalf("unsupported value %T %v", kv.Value, kv.Value) } @@ -261,30 +306,68 @@ func getFuncPackage(fun ast.Expr) string { if iexpr, ok := fun.(*ast.IndexExpr); ok { return getFuncPackage(iexpr.X) } + if _, ok := fun.(*ast.FuncLit); ok { + return "" + } log.Fatalf("unsupported func expression %T, %v", fun, fun) return "" } +func getBinaryExpr(b *ast.BinaryExpr) string { + var x, y string + switch val := b.X.(type) { + case *ast.BasicLit: + x = strings.Trim(val.Value, `"`) + case *ast.BinaryExpr: + x = getBinaryExpr(val) + default: + log.Fatalf("unsupported value %T %v", val, val) + } + switch val := b.Y.(type) { + case *ast.BasicLit: + y = strings.Trim(val.Value, `"`) + case *ast.BinaryExpr: + y = getBinaryExpr(val) + default: + log.Fatalf("unsupported value %T %v", val, val) + } + return x + y +} + // we cannot get the value of an Identifier directly so we map it manually instead func getIdentMapping(identName string) (string, error) { identMapping := map[string]string{ "metrics.Namespace": metrics.Namespace, "Namespace": metrics.Namespace, - "NodeSubsystem": "nodes", - "metrics.NodeSubsystem": "nodes", - "machineSubsystem": "machines", - "nodeClaimSubsystem": "nodeclaims", + "MetricNamespace": "operator", + "MetricSubsystem": "status_condition", + "WorkQueueSubsystem": "workqueue", + "DepthKey": "depth", + "AddsKey": "adds_total", + "QueueLatencyKey": "queue_duration_seconds", + "WorkDurationKey": "work_duration_seconds", + "UnfinishedWorkKey": "unfinished_work_seconds", + "LongestRunningProcessorKey": "longest_running_processor_seconds", + "RetriesKey": "retries_total", + + "metrics.PodSubsystem": "pods", + "NodeSubsystem": "nodes", + "metrics.NodeSubsystem": "nodes", + "machineSubsystem": "machines", + "NodeClaimSubsystem": "nodeclaims", + "metrics.NodeClaimSubsystem": "nodeclaims", // TODO @joinnis: We should eventually change this subsystem to be // plural so that it aligns with the other subsystems - "nodePoolSubsystem": "nodepool", - "interruptionSubsystem": "interruption", - "deprovisioningSubsystem": "deprovisioning", - "disruptionSubsystem": "disruption", - "consistencySubsystem": "consistency", - "batcherSubsystem": "cloudprovider_batcher", - "cloudProviderSubsystem": "cloudprovider", - "stateSubsystem": "cluster_state", + "nodePoolSubsystem": "nodepools", + "metrics.NodePoolSubsystem": "nodepools", + "interruptionSubsystem": "interruption", + "deprovisioningSubsystem": "deprovisioning", + "voluntaryDisruptionSubsystem": "voluntary_disruption", + "batcherSubsystem": "cloudprovider_batcher", + "cloudProviderSubsystem": "cloudprovider", + "stateSubsystem": "cluster_state", + "schedulerSubsystem": "scheduler", } if v, ok := identMapping[identName]; ok { return v, nil diff --git a/hack/docs/version_compatibility.go b/hack/docs/version_compatibility_gen/main.go similarity index 100% rename from hack/docs/version_compatibility.go rename to hack/docs/version_compatibility_gen/main.go diff --git a/hack/mutation/conversion_webhooks_injection.sh b/hack/mutation/conversion_webhooks_injection.sh new file mode 100755 index 000000000000..1a466499d041 --- /dev/null +++ b/hack/mutation/conversion_webhooks_injection.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Add the conversion stanza to the CRD spec to enable conversion via webhook +yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig": {"service": {"name": "karpenter", "namespace": "kube-system", "port": 8443}}}}' -i pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig": {"service": {"name": "karpenter", "namespace": "kube-system", "port": 8443}}}}' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml +yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig": {"service": {"name": "karpenter", "namespace": "kube-system", "port": 8443}}}}' -i pkg/apis/crds/karpenter.sh_nodepools.yaml + +# Update to the karpenter-crd charts + +# Add the conversion stanza to the CRD spec to enable conversion via webhook +echo "{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace }} + port: {{ .Values.webhook.port }} +{{- end }} +" >> charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml + +echo "{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace }} + port: {{ .Values.webhook.port }} +{{- end }} +" >> charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml + +echo "{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace }} + port: {{ .Values.webhook.port }} +{{- end }} +" >> charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml \ No newline at end of file diff --git a/hack/validation/kubelet.sh b/hack/validation/kubelet.sh new file mode 100755 index 000000000000..621b1ac2267e --- /dev/null +++ b/hack/validation/kubelet.sh @@ -0,0 +1,13 @@ +# Kubelet Validation + +# The regular expression adds validation for kubelet.kubeReserved and kubelet.systemReserved values of the map are resource.Quantity +# Quantity: https://github.com/kubernetes/apimachinery/blob/d82afe1e363acae0e8c0953b1bc230d65fdb50e2/pkg/api/resource/quantity.go#L100 +# EC2NodeClass Validation: +yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.kubeReserved.additionalProperties.pattern = "^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$"' -i pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.systemReserved.additionalProperties.pattern = "^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$"' -i pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml + +# The regular expression is a validation for kubelet.evictionHard and kubelet.evictionSoft are percentage or a resource.Quantity +# Quantity: https://github.com/kubernetes/apimachinery/blob/d82afe1e363acae0e8c0953b1bc230d65fdb50e2/pkg/api/resource/quantity.go#L100 +# EC2NodeClass Validation: +yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.evictionHard.additionalProperties.pattern = "^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$"' -i pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.evictionSoft.additionalProperties.pattern = "^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$"' -i pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml diff --git a/hack/validation/labels.sh b/hack/validation/labels.sh index 1fb9c6293b32..18257ba048c5 100755 --- a/hack/validation/labels.sh +++ b/hack/validation/labels.sh @@ -4,4 +4,8 @@ # ## checking for restricted labels while filtering out well known labels yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.template.properties.metadata.properties.labels.x-kubernetes-validations += [ - {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self.all(x, x in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !x.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\"))"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file + {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self.all(x, x in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-ebs-bandwidth\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !x.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\"))"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml + +# ## checking for restricted labels while filtering out well known labels +yq eval '.spec.versions[1].schema.openAPIV3Schema.properties.spec.properties.template.properties.metadata.properties.labels.x-kubernetes-validations += [ + {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self.all(x, x in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-ebs-bandwidth\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !x.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\"))"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file diff --git a/hack/validation/requirements.sh b/hack/validation/requirements.sh index 763d359fab16..69a1c0b68c99 100755 --- a/hack/validation/requirements.sh +++ b/hack/validation/requirements.sh @@ -2,11 +2,22 @@ # Adding validation for nodeclaim +# v1beta1 +## checking for restricted labels while filtering out well known labels +yq eval '.spec.versions[1].schema.openAPIV3Schema.properties.spec.properties.requirements.items.properties.key.x-kubernetes-validations += [ + {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-ebs-bandwidth\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml +# # Adding validation for nodepool + +# ## checking for restricted labels while filtering out well known labels +yq eval '.spec.versions[1].schema.openAPIV3Schema.properties.spec.properties.template.properties.spec.properties.requirements.items.properties.key.x-kubernetes-validations += [ + {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-ebs-bandwidth\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml + +# v1 ## checking for restricted labels while filtering out well known labels yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.requirements.items.properties.key.x-kubernetes-validations += [ - {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml + {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-ebs-bandwidth\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml # # Adding validation for nodepool # ## checking for restricted labels while filtering out well known labels yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.template.properties.spec.properties.requirements.items.properties.key.x-kubernetes-validations += [ - {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml + {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-ebs-bandwidth\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml diff --git a/pkg/apis/apis.go b/pkg/apis/apis.go index f99bd99dd8f6..3e255ad9275d 100644 --- a/pkg/apis/apis.go +++ b/pkg/apis/apis.go @@ -18,31 +18,19 @@ package apis import ( _ "embed" - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/runtime" - - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" - - "github.com/samber/lo" + "github.com/awslabs/operatorpkg/object" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "sigs.k8s.io/karpenter/pkg/apis" - "sigs.k8s.io/karpenter/pkg/utils/functional" -) - -var ( - // Builder includes all types within the apis package - Builder = runtime.NewSchemeBuilder( - v1beta1.SchemeBuilder.AddToScheme, - ) - // AddToScheme may be used to add all resources defined in the project to a Scheme - AddToScheme = Builder.AddToScheme ) //go:generate controller-gen crd object:headerFile="../../hack/boilerplate.go.txt" paths="./..." output:crd:artifacts:config=crds var ( + Group = "karpenter.k8s.aws" + CompatibilityGroup = "compatibility." + Group //go:embed crds/karpenter.k8s.aws_ec2nodeclasses.yaml EC2NodeClassCRD []byte CRDs = append(apis.CRDs, - lo.Must(functional.Unmarshal[v1.CustomResourceDefinition](EC2NodeClassCRD)), + object.Unmarshal[apiextensionsv1.CustomResourceDefinition](EC2NodeClassCRD), ) ) diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index fe04920f8dad..09a8d1e9dba0 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -3,568 +3,1344 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: ec2nodeclasses.karpenter.k8s.aws spec: group: karpenter.k8s.aws names: categories: - - karpenter + - karpenter kind: EC2NodeClass listKind: EC2NodeClassList plural: ec2nodeclasses shortNames: - - ec2nc - - ec2ncs + - ec2nc + - ec2ncs singular: ec2nodeclass scope: Cluster versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: EC2NodeClass is the Schema for the EC2NodeClass API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. - This will contain configuration necessary to launch instances in AWS. - properties: - amiFamily: - description: AMIFamily is the AMI family that instances use. - enum: - - AL2 - - AL2023 - - Bottlerocket - - Ubuntu - - Custom - - Windows2019 - - Windows2022 - type: string - amiSelectorTerms: - description: AMISelectorTerms is a list of or ami selector terms. - The terms are ORed. - items: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.role + name: Role + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: EC2NodeClass is the Schema for the EC2NodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. + properties: + amiFamily: description: |- - AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. - If multiple fields are used for selection, the requirements are ANDed. - properties: - id: - description: ID is the ami id in EC2 - pattern: ami-[0-9a-z]+ - type: string - name: - description: |- - Name is the ami name in EC2. - This value is the name field, which is different from the name tag. - type: string - owner: - description: |- - Owner is the owner for the ami. - You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" - type: string - tags: - additionalProperties: + AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + alias is specified, this field is required. + NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + the AMIFamily() helper function + enum: + - AL2 + - AL2023 + - Bottlerocket + - Custom + - Windows2019 + - Windows2022 + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + alias: + description: |- + Alias specifies which EKS optimized AMI to select. + Each alias consists of a family and an AMI version, specified as "family@version". + Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + Note: The Windows families do **not** support version pinning, and only latest may be used. + maxLength: 30 type: string - description: |- - Tags is a map of key/value tags used to select subnets - Specifying '*' for a value selects all values for a given tag key. - maxProperties: 20 - type: object - x-kubernetes-validations: - - message: empty tag keys or values aren't supported - rule: self.all(k, k != '' && self[k] != '') - type: object - maxItems: 30 - type: array - x-kubernetes-validations: - - message: expected at least one, got none, ['tags', 'id', 'name'] - rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) - - message: '''id'' is mutually exclusive, cannot be set with a combination - of other fields in amiSelectorTerms' - rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name) || - has(x.owner)))' - associatePublicIPAddress: - description: AssociatePublicIPAddress controls if public IP addresses - are assigned to instances that are launched with the nodeclass. - type: boolean - blockDeviceMappings: - description: BlockDeviceMappings to be applied to provisioned nodes. - items: - properties: - deviceName: - description: The device name (for example, /dev/sdh or xvdh). - type: string - ebs: - description: EBS contains parameters used to automatically set - up EBS volumes when an instance is launched. - properties: - deleteOnTermination: - description: DeleteOnTermination indicates whether the EBS - volume is deleted on instance termination. - type: boolean - encrypted: - description: |- - Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only - be attached to instances that support Amazon EBS encryption. If you are creating - a volume from a snapshot, you can't specify an encryption value. - type: boolean - iops: - description: |- - IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, - this represents the number of IOPS that are provisioned for the volume. For - gp2 volumes, this represents the baseline performance of the volume and the - rate at which the volume accumulates I/O credits for bursting. + x-kubernetes-validations: + - message: '''alias'' is improperly formatted, must match the format ''family@version''' + rule: self.matches('^[a-zA-Z0-9]+@.+$') + - message: 'family is not supported, must be one of the following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', ''windows2022''' + rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022'] + - message: windows families may only specify version 'latest' + rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022''] ? self.split(''@'')[1] == ''latest'' : true' + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + minItems: 1 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name', 'alias'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms' + rule: '!(self.exists(x, has(x.alias)) && self.size() != 1)' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. - The following are the supported values for each volume type: + The following are the supported values for each volume type: - * gp3: 3,000-16,000 IOPS + * gp3: 3,000-16,000 IOPS - * io1: 100-64,000 IOPS + * io1: 100-64,000 IOPS - * io2: 100-64,000 IOPS + * io2: 100-64,000 IOPS - For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built - on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). - Other instance families guarantee performance up to 32,000 IOPS. + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. - This parameter is supported for io1, io2, and gp3 volumes only. This parameter - is not supported for gp2, st1, sc1, or standard volumes. - format: int64 - type: integer - kmsKeyID: - description: KMSKeyID (ARN) of the symmetric Key Management - Service (KMS) CMK used for encryption. - type: string - snapshotID: - description: SnapshotID is the ID of an EBS snapshot - type: string - throughput: - description: |- - Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. - Valid Range: Minimum value of 125. Maximum value of 1000. - format: int64 - type: integer - volumeSize: - allOf: - - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - - pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ - anyOf: - - type: integer - - type: string - description: |- - VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or - a volume size. The following are the supported volumes sizes for each volume - type: + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: KMSKeyID (ARN) of the symmetric Key Management Service (KMS) CMK used for encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: - * gp2 and gp3: 1-16,384 + * gp2 and gp3: 1-16,384 - * io1 and io2: 4-16,384 + * io1 and io2: 4-16,384 - * st1 and sc1: 125-16,384 + * st1 and sc1: 125-16,384 - * standard: 1-1,024 - x-kubernetes-int-or-string: true - volumeType: - description: |- - VolumeType of the block device. - For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) - in the Amazon Elastic Compute Cloud User Guide. - enum: - - standard - - io1 - - io2 - - gp2 - - sc1 - - st1 - - gp3 - type: string + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store disks. + enum: + - RAID0 + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds type: object x-kubernetes-validations: - - message: snapshotID or volumeSize must be defined - rule: has(self.snapshotID) || has(self.volumeSize) - rootVolume: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: description: |- - RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can - configure at most one root volume in BlockDeviceMappings. - type: boolean + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) type: object - maxItems: 50 - type: array - x-kubernetes-validations: - - message: must have only one blockDeviceMappings with rootVolume - rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() - <= 1 - context: - description: |- - Context is a Reserved field in EC2 APIs - https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html - type: string - detailedMonitoring: - description: DetailedMonitoring controls if detailed monitoring is - enabled for instances that are launched - type: boolean - instanceProfile: - description: |- - InstanceProfile is the AWS entity that instances use. - This field is mutually exclusive from role. - The instance profile should already have a role assigned to it that Karpenter - has PassRole permission on for instance launch using this instanceProfile to succeed. - type: string - x-kubernetes-validations: - - message: instanceProfile cannot be empty - rule: self != '' - instanceStorePolicy: - description: InstanceStorePolicy specifies how to handle instance-store - disks. - enum: - - RAID0 - type: string - metadataOptions: - default: - httpEndpoint: enabled - httpProtocolIPv6: disabled - httpPutResponseHopLimit: 2 - httpTokens: required - description: |- - MetadataOptions for the generated launch template of provisioned nodes. - - - This specifies the exposure of the Instance Metadata Service to - provisioned EC2 nodes. For more information, - see Instance Metadata and User Data - (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) - in the Amazon Elastic Compute Cloud User Guide. - - - Refer to recommended, security best practices - (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) - for limiting exposure of Instance Metadata and User Data to pods. - If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 - disabled, with httpPutResponseLimit of 2, and with httpTokens - required. - properties: - httpEndpoint: - default: enabled - description: |- - HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned - nodes. If metadata options is non-nil, but this parameter is not specified, - the default state is "enabled". + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. - If you specify a value of "disabled", instance metadata will not be accessible - on the node. - enum: - - enabled - - disabled - type: string - httpProtocolIPv6: - default: disabled - description: |- - HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata - service on provisioned nodes. If metadata options is non-nil, but this parameter - is not specified, the default state is "disabled". - enum: - - enabled - - disabled - type: string - httpPutResponseHopLimit: - default: 2 - description: |- - HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for - instance metadata requests. The larger the number, the further instance - metadata requests can travel. Possible values are integers from 1 to 64. - If metadata options is non-nil, but this parameter is not specified, the - default value is 2. - format: int64 - maximum: 64 - minimum: 1 - type: integer - httpTokens: - default: required - description: |- - HTTPTokens determines the state of token usage for instance metadata - requests. If metadata options is non-nil, but this parameter is not - specified, the default state is "required". + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. - If the state is optional, one can choose to retrieve instance metadata with - or without a signed token header on the request. If one retrieves the IAM - role credentials without a token, the version 1.0 role credentials are - returned. If one retrieves the IAM role credentials using a valid signed - token, the version 2.0 role credentials are returned. + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. + properties: + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". - If the state is "required", one must send a signed token header with any - instance metadata retrieval requests. In this state, retrieving the IAM - role credentials always returns the version 2.0 credentials; the version - 1.0 credentials are not available. - enum: - - required - - optional - type: string - type: object - role: - description: |- - Role is the AWS identity that nodes use. This field is immutable. - This field is mutually exclusive from instanceProfile. - Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. - This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented - for the old instance profiles on an update. - type: string - x-kubernetes-validations: - - message: role cannot be empty - rule: self != '' - - message: immutable field changed - rule: self == oldSelf - securityGroupSelectorTerms: - description: SecurityGroupSelectorTerms is a list of or security group - selector terms. The terms are ORed. - items: - description: |- - SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. - If multiple fields are used for selection, the requirements are ANDed. - properties: - id: - description: ID is the security group id in EC2 - pattern: sg-[0-9a-z]+ + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled type: string - name: + httpProtocolIPv6: + default: disabled description: |- - Name is the security group name in EC2. - This value is the name field, which is different from the name tag. + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled type: string - tags: - additionalProperties: - type: string + httpPutResponseHopLimit: + default: 1 description: |- - Tags is a map of key/value tags used to select subnets - Specifying '*' for a value selects all values for a given tag key. - maxProperties: 20 - type: object - x-kubernetes-validations: - - message: empty tag keys or values aren't supported - rule: self.all(k, k != '' && self[k] != '') + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 1. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional + type: string type: object - maxItems: 30 - type: array - x-kubernetes-validations: - - message: securityGroupSelectorTerms cannot be empty - rule: self.size() != 0 - - message: expected at least one, got none, ['tags', 'id', 'name'] - rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) - - message: '''id'' is mutually exclusive, cannot be set with a combination - of other fields in securityGroupSelectorTerms' - rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' - - message: '''name'' is mutually exclusive, cannot be set with a combination - of other fields in securityGroupSelectorTerms' - rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' - subnetSelectorTerms: - description: SubnetSelectorTerms is a list of or subnet selector terms. - The terms are ORed. - items: + role: description: |- - SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. - If multiple fields are used for selection, the requirements are ANDed. - properties: - id: - description: ID is the subnet id in EC2 - pattern: subnet-[0-9a-z]+ - type: string - tags: - additionalProperties: + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ type: string - description: |- - Tags is a map of key/value tags used to select subnets - Specifying '*' for a value selects all values for a given tag key. - maxProperties: 20 - type: object - x-kubernetes-validations: - - message: empty tag keys or values aren't supported - rule: self.all(k, k != '' && self[k] != '') + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in subnetSelectorTerms' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances and launch templates. type: object - maxItems: 30 - type: array - x-kubernetes-validations: - - message: subnetSelectorTerms cannot be empty - rule: self.size() != 0 - - message: expected at least one, got none, ['tags', 'id'] - rule: self.all(x, has(x.tags) || has(x.id)) - - message: '''id'' is mutually exclusive, cannot be set with a combination - of other fields in subnetSelectorTerms' - rule: '!self.all(x, has(x.id) && has(x.tags))' - tags: - additionalProperties: + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. type: string - description: Tags to be applied on ec2 resources like instances and - launch templates. - type: object - x-kubernetes-validations: - - message: empty tag keys aren't supported - rule: self.all(k, k != '') - - message: tag contains a restricted tag matching kubernetes.io/cluster/ - rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) - - message: tag contains a restricted tag matching karpenter.sh/nodepool - rule: self.all(k, k != 'karpenter.sh/nodepool') - - message: tag contains a restricted tag matching karpenter.sh/managed-by - rule: self.all(k, k !='karpenter.sh/managed-by') - - message: tag contains a restricted tag matching karpenter.sh/nodeclaim - rule: self.all(k, k !='karpenter.sh/nodeclaim') - - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass - rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') - userData: - description: |- - UserData to be applied to the provisioned nodes. - It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into - this UserData to ensure nodes are being provisioned with the correct configuration. - type: string - required: - - amiFamily - - securityGroupSelectorTerms - - subnetSelectorTerms - type: object - x-kubernetes-validations: - - message: amiSelectorTerms is required when amiFamily == 'Custom' - rule: 'self.amiFamily == ''Custom'' ? self.amiSelectorTerms.size() != - 0 : true' - - message: must specify exactly one of ['role', 'instanceProfile'] - rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) - && has(self.instanceProfile)) - - message: changing from 'instanceProfile' to 'role' is not supported. - You must delete and recreate this node class if you want to change - this. - rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) - && has(self.instanceProfile)) - status: - description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass - properties: - amis: - description: |- - AMI contains the current AMI values that are available to the - cluster under the AMI selectors. - items: - description: AMI contains resolved AMI selector values utilized - for node launch - properties: - id: - description: ID of the AMI - type: string - name: - description: Name of the AMI - type: string - requirements: - description: Requirements of the AMI to be utilized on an instance - type - items: + required: + - amiSelectorTerms + - securityGroupSelectorTerms + - subnetSelectorTerms + type: object + x-kubernetes-validations: + - message: must specify exactly one of ['role', 'instanceProfile'] + rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile)) + - message: changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this. + rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile)) + - message: if set, amiFamily must be 'AL2' or 'Custom' when using an AL2 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2'') : true)' + - message: if set, amiFamily must be 'AL2023' or 'Custom' when using an AL2023 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2023'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2023'') : true)' + - message: if set, amiFamily must be 'Bottlerocket' or 'Custom' when using a Bottlerocket alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''bottlerocket'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Bottlerocket'') : true)' + - message: if set, amiFamily must be 'Windows2019' or 'Custom' when using a Windows2019 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2019'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2019'') : true)' + - message: if set, amiFamily must be 'Windows2022' or 'Custom' when using a Windows2022 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2022'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2022'') : true)' + - message: must specify amiFamily if amiSelectorTerms does not contain an alias + rule: 'self.amiSelectorTerms.exists(x, has(x.alias)) ? true : has(self.amiFamily)' + status: + description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass + properties: + amis: + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. + items: + description: AMI contains resolved AMI selector values utilized for node launch + properties: + id: + description: ID of the AMI + type: string + name: + description: Name of the AMI + type: string + requirements: + description: Requirements of the AMI to be utilized on an instance type + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + required: + - id + - requirements + type: object + type: array + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + instanceProfile: + description: InstanceProfile contains the resolved instance profile for the role + type: string + securityGroups: + description: |- + SecurityGroups contains the current Security Groups values that are available to the + cluster under the SecurityGroups selectors. + items: + description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch + properties: + id: + description: ID of the security group + type: string + name: + description: Name of the security group + type: string + required: + - id + type: object + type: array + subnets: + description: |- + Subnets contains the current Subnet values that are available to the + cluster under the subnet selectors. + items: + description: Subnet contains resolved Subnet selector values utilized for node launch + properties: + id: + description: ID of the subnet + type: string + zone: + description: The associated availability zone + type: string + zoneID: + description: The associated availability zone ID + type: string + required: + - id + - zone + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} + - name: v1beta1 + schema: + openAPIV3Schema: + description: EC2NodeClass is the Schema for the EC2NodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. + properties: + amiFamily: + description: AMIFamily is the AMI family that instances use. + enum: + - AL2 + - AL2023 + - Bottlerocket + - Ubuntu + - Custom + - Windows2019 + - Windows2022 + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + tags: + additionalProperties: + type: string description: |- - A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values - and minValues that represent the requirement to have at least that many values. + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name) || has(x.owner)))' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically set up EBS volumes when an instance is launched. properties: - key: - description: The label key that the selector applies to. + deleteOnTermination: + description: DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + + The following are the supported values for each volume type: + + + * gp3: 3,000-16,000 IOPS + + + * io1: 100-64,000 IOPS + + + * io2: 100-64,000 IOPS + + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: KMSKeyID (ARN) of the symmetric Key Management Service (KMS) CMK used for encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot type: string - minValues: + throughput: description: |- - This field is ALPHA and can be dropped or replaced at any time - MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. - maximum: 50 - minimum: 1 + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 type: integer - operator: + volumeSize: description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + + * gp2 and gp3: 1-16,384 + + + * io1 and io2: 4-16,384 + + + * st1 and sc1: 125-16,384 + + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ type: string - values: + volumeType: description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string type: object - type: array - required: - - id - - requirements - type: object - type: array - instanceProfile: - description: InstanceProfile contains the resolved instance profile - for the role - type: string - securityGroups: - description: |- - SecurityGroups contains the current Security Groups values that are available to the - cluster under the SecurityGroups selectors. - items: - description: SecurityGroup contains resolved SecurityGroup selector - values utilized for node launch + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store disks. + enum: + - RAID0 + type: string + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. properties: - id: - description: ID of the security group - type: string - name: - description: Name of the security group + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled type: string - required: - - id - type: object - type: array - subnets: - description: |- - Subnets contains the current Subnet values that are available to the - cluster under the subnet selectors. - items: - description: Subnet contains resolved Subnet selector values utilized - for node launch - properties: - id: - description: ID of the subnet + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled type: string - zone: - description: The associated availability zone + httpPutResponseHopLimit: + default: 2 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 2. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional type: string - required: - - id - - zone type: object - type: array - type: object - type: object - served: true - storage: true - subresources: - status: {} + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in subnetSelectorTerms' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances and launch templates. + type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/managed-by + rule: self.all(k, k !='karpenter.sh/managed-by') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiFamily + - securityGroupSelectorTerms + - subnetSelectorTerms + type: object + x-kubernetes-validations: + - message: amiSelectorTerms is required when amiFamily == 'Custom' + rule: 'self.amiFamily == ''Custom'' ? self.amiSelectorTerms.size() != 0 : true' + - message: must specify exactly one of ['role', 'instanceProfile'] + rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile)) + - message: changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this. + rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile)) + status: + description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass + properties: + amis: + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. + items: + description: AMI contains resolved AMI selector values utilized for node launch + properties: + id: + description: ID of the AMI + type: string + name: + description: Name of the AMI + type: string + requirements: + description: Requirements of the AMI to be utilized on an instance type + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + required: + - id + - requirements + type: object + type: array + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + instanceProfile: + description: InstanceProfile contains the resolved instance profile for the role + type: string + securityGroups: + description: |- + SecurityGroups contains the current Security Groups values that are available to the + cluster under the SecurityGroups selectors. + items: + description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch + properties: + id: + description: ID of the security group + type: string + name: + description: Name of the security group + type: string + required: + - id + type: object + type: array + subnets: + description: |- + Subnets contains the current Subnet values that are available to the + cluster under the subnet selectors. + items: + description: Subnet contains resolved Subnet selector values utilized for node launch + properties: + id: + description: ID of the subnet + type: string + zone: + description: The associated availability zone + type: string + zoneID: + description: The associated availability zone ID + type: string + required: + - id + - zone + type: object + type: array + type: object + type: object + served: true + storage: false + subresources: + status: {} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: karpenter + namespace: kube-system + port: 8443 diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml index 37abdeca8e80..064e0dad59bc 100644 --- a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: nodeclaims.karpenter.sh spec: group: karpenter.sh @@ -16,6 +16,379 @@ spec: singular: nodeclaim scope: Cluster versions: + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + type: string + - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.providerID + name: ID + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool + priority: 1 + type: string + - jsonPath: .spec.nodeClassRef.name + name: NodeClass + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + resources: + description: Resources models the resource requirements for the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object + type: object + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + x-kubernetes-validations: + - message: spec is immutable + rule: self == oldSelf + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + imageID: + description: ImageID is an identifier for the image that runs on the node + type: string + lastPodEventTime: + description: |- + LastPodEventTime is updated with the last time a pod was scheduled + or removed from the node. A pod going terminal or terminating + is also considered as removed. + format: date-time + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} - additionalPrinterColumns: - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type name: Type @@ -220,7 +593,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time @@ -250,13 +623,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic maxLength: 63 pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ required: - key - operator type: object - maxItems: 30 + maxItems: 100 type: array x-kubernetes-validations: - message: requirements with operator 'In' must have a value defined @@ -384,34 +758,57 @@ spec: conditions: description: Conditions contains signals for health and readiness items: - description: |- - Condition defines a readiness condition for a Knative resource. - See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties + description: Condition aliases the upstream type and adds additional helper methods properties: lastTransitionTime: description: |- - LastTransitionTime is the last time the condition transitioned from one status to another. - We use VolatileTime in place of metav1.Time to exclude this from creating equality.Semantic - differences (all other things held constant). + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time type: string message: - description: A human readable message indicating details about the transition. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer reason: - description: The reason for the condition's last transition. - type: string - severity: description: |- - Severity with which to treat failures of this type of condition. - When this is not specified, it defaults to Error. + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ type: string status: - description: Status of the condition, one of True, False, Unknown. + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown type: string type: - description: Type of condition. + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: + - lastTransitionTime - status - type type: object @@ -430,6 +827,17 @@ spec: - spec type: object served: true - storage: true + storage: false subresources: status: {} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: karpenter + namespace: kube-system + port: 8443 diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index 44a38267e3e9..3e5b2bb2888d 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: nodepools.karpenter.sh spec: group: karpenter.sh @@ -16,6 +16,500 @@ spec: singular: nodepool scope: Cluster versions: + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .status.resources.nodes + name: Nodes + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: integer + - jsonPath: .status.resources.cpu + name: CPU + priority: 1 + type: string + - jsonPath: .status.resources.memory + name: Memory + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. + properties: + disruption: + description: Disruption contains the parameters that relate to Karpenter's disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons for disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + consolidationPolicy: + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized + type: string + required: + - consolidateAfter + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + maxProperties: 100 + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + - message: label domain "karpenter.k8s.aws" is restricted + rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + required: + - spec + type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} - additionalPrinterColumns: - jsonPath: .spec.template.spec.nodeClassRef.name name: NodeClass @@ -190,7 +684,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self.all(x, x != "kubernetes.io/hostname") - message: label domain "karpenter.k8s.aws" is restricted - rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) type: object spec: description: NodeClaimSpec describes the desired state of the NodeClaim @@ -348,7 +842,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time @@ -378,13 +872,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic maxLength: 63 pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ required: - key - operator type: object - maxItems: 30 + maxItems: 100 type: array x-kubernetes-validations: - message: requirements with operator 'In' must have a value defined @@ -508,6 +1003,67 @@ spec: status: description: NodePoolStatus defines the observed state of NodePool properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array resources: additionalProperties: anyOf: @@ -522,6 +1078,17 @@ spec: - spec type: object served: true - storage: true + storage: false subresources: status: {} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: karpenter + namespace: kube-system + port: 8443 diff --git a/pkg/apis/v1beta1/register.go b/pkg/apis/v1/doc.go similarity index 53% rename from pkg/apis/v1beta1/register.go rename to pkg/apis/v1/doc.go index b46161bbdc6c..44692b28c362 100644 --- a/pkg/apis/v1beta1/register.go +++ b/pkg/apis/v1/doc.go @@ -12,24 +12,25 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen=package,register +// +k8s:defaulter-gen=TypeMeta +// +groupName=karpenter.k8s.aws +package v1 // doc.go is discovered by codegen import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" + corev1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" -) - -const Group = "karpenter.k8s.aws" + "k8s.io/client-go/kubernetes/scheme" -var ( - SchemeGroupVersion = schema.GroupVersion{Group: Group, Version: "v1beta1"} - SchemeBuilder = runtime.NewSchemeBuilder(func(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, - &EC2NodeClass{}, - &EC2NodeClassList{}, - ) - metav1.AddToGroupVersion(scheme, SchemeGroupVersion) - return nil - }) + "github.com/aws/karpenter-provider-aws/pkg/apis" ) + +func init() { + gv := schema.GroupVersion{Group: apis.Group, Version: "v1"} + corev1.AddToGroupVersion(scheme.Scheme, gv) + scheme.Scheme.AddKnownTypes(gv, + &EC2NodeClass{}, + &EC2NodeClassList{}, + ) +} diff --git a/pkg/apis/v1/ec2nodeclass.go b/pkg/apis/v1/ec2nodeclass.go new file mode 100644 index 000000000000..5ad59c7d8552 --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass.go @@ -0,0 +1,535 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "fmt" + "log" + "strings" + + "github.com/mitchellh/hashstructure/v2" + "github.com/samber/lo" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. +// This will contain configuration necessary to launch instances in AWS. +type EC2NodeClassSpec struct { + // SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="subnetSelectorTerms cannot be empty",rule="self.size() != 0" + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id']",rule="self.all(x, has(x.tags) || has(x.id))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in subnetSelectorTerms",rule="!self.all(x, has(x.id) && has(x.tags))" + // +kubebuilder:validation:MaxItems:=30 + // +required + SubnetSelectorTerms []SubnetSelectorTerm `json:"subnetSelectorTerms" hash:"ignore"` + // SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="securityGroupSelectorTerms cannot be empty",rule="self.size() != 0" + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms",rule="!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))" + // +kubebuilder:validation:XValidation:message="'name' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms",rule="!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))" + // +kubebuilder:validation:MaxItems:=30 + // +required + SecurityGroupSelectorTerms []SecurityGroupSelectorTerm `json:"securityGroupSelectorTerms" hash:"ignore"` + // AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + // +optional + AssociatePublicIPAddress *bool `json:"associatePublicIPAddress,omitempty"` + // AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name', 'alias']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))" + // +kubebuilder:validation:XValidation:message="'alias' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) || has(x.name) || has(x.owner)))" + // +kubebuilder:validation:XValidation:message="'alias' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms",rule="!(self.exists(x, has(x.alias)) && self.size() != 1)" + // +kubebuilder:validation:MinItems:=1 + // +kubebuilder:validation:MaxItems:=30 + // +required + AMISelectorTerms []AMISelectorTerm `json:"amiSelectorTerms" hash:"ignore"` + // AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + // This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + // family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + // alias is specified, this field is required. + // NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + // the AMIFamily() helper function + // +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Custom,Windows2019,Windows2022} + // +optional + AMIFamily *string `json:"amiFamily,omitempty" hash:"ignore"` + // UserData to be applied to the provisioned nodes. + // It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + // this UserData to ensure nodes are being provisioned with the correct configuration. + // +optional + UserData *string `json:"userData,omitempty"` + // Role is the AWS identity that nodes use. This field is immutable. + // This field is mutually exclusive from instanceProfile. + // Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + // This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + // for the old instance profiles on an update. + // +kubebuilder:validation:XValidation:rule="self != ''",message="role cannot be empty" + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="immutable field changed" + // +optional + Role string `json:"role,omitempty"` + // InstanceProfile is the AWS entity that instances use. + // This field is mutually exclusive from role. + // The instance profile should already have a role assigned to it that Karpenter + // has PassRole permission on for instance launch using this instanceProfile to succeed. + // +kubebuilder:validation:XValidation:rule="self != ''",message="instanceProfile cannot be empty" + // +optional + InstanceProfile *string `json:"instanceProfile,omitempty"` + // Tags to be applied on ec2 resources like instances and launch templates. + // +kubebuilder:validation:XValidation:message="empty tag keys aren't supported",rule="self.all(k, k != '')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching eks:eks-cluster-name",rule="self.all(k, k !='eks:eks-cluster-name')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching kubernetes.io/cluster/",rule="self.all(k, !k.startsWith('kubernetes.io/cluster') )" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/nodepool",rule="self.all(k, k != 'karpenter.sh/nodepool')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/nodeclaim",rule="self.all(k, k !='karpenter.sh/nodeclaim')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass",rule="self.all(k, k !='karpenter.k8s.aws/ec2nodeclass')" + // +optional + Tags map[string]string `json:"tags,omitempty"` + // Kubelet defines args to be used when configuring kubelet on provisioned nodes. + // They are a subset of the upstream types, recognizing not all options may be supported. + // Wherever possible, the types and names should reflect the upstream kubelet types. + // +kubebuilder:validation:XValidation:message="imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent",rule="has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true" + // +kubebuilder:validation:XValidation:message="evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod",rule="has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true" + // +kubebuilder:validation:XValidation:message="evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft",rule="has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true" + // +optional + Kubelet *KubeletConfiguration `json:"kubelet,omitempty" hash:"ignore"` + // BlockDeviceMappings to be applied to provisioned nodes. + // +kubebuilder:validation:XValidation:message="must have only one blockDeviceMappings with rootVolume",rule="self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1" + // +kubebuilder:validation:MaxItems:=50 + // +optional + BlockDeviceMappings []*BlockDeviceMapping `json:"blockDeviceMappings,omitempty"` + // InstanceStorePolicy specifies how to handle instance-store disks. + // +optional + InstanceStorePolicy *InstanceStorePolicy `json:"instanceStorePolicy,omitempty"` + // DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + // +optional + DetailedMonitoring *bool `json:"detailedMonitoring,omitempty"` + // MetadataOptions for the generated launch template of provisioned nodes. + // + // This specifies the exposure of the Instance Metadata Service to + // provisioned EC2 nodes. For more information, + // see Instance Metadata and User Data + // (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + // in the Amazon Elastic Compute Cloud User Guide. + // + // Refer to recommended, security best practices + // (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + // for limiting exposure of Instance Metadata and User Data to pods. + // If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + // disabled, with httpPutResponseLimit of 1, and with httpTokens + // required. + // +kubebuilder:default={"httpEndpoint":"enabled","httpProtocolIPv6":"disabled","httpPutResponseHopLimit":1,"httpTokens":"required"} + // +optional + MetadataOptions *MetadataOptions `json:"metadataOptions,omitempty"` + // Context is a Reserved field in EC2 APIs + // https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + // +optional + Context *string `json:"context,omitempty"` +} + +// SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type SubnetSelectorTerm struct { + // Tags is a map of key/value tags used to select subnets + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the subnet id in EC2 + // +kubebuilder:validation:Pattern="subnet-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` +} + +// SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type SecurityGroupSelectorTerm struct { + // Tags is a map of key/value tags used to select subnets + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the security group id in EC2 + // +kubebuilder:validation:Pattern:="sg-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` + // Name is the security group name in EC2. + // This value is the name field, which is different from the name tag. + Name string `json:"name,omitempty"` +} + +// AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type AMISelectorTerm struct { + // Alias specifies which EKS optimized AMI to select. + // Each alias consists of a family and an AMI version, specified as "family@version". + // Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + // The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + // The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + // Note: The Windows families do **not** support version pinning, and only latest may be used. + // +kubebuilder:validation:XValidation:message="'alias' is improperly formatted, must match the format 'family@version'",rule="self.matches('^[a-zA-Z0-9]+@.+$')" + // +kubebuilder:validation:XValidation:message="family is not supported, must be one of the following: 'al2', 'al2023', 'bottlerocket', 'windows2019', 'windows2022'",rule="self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022']" + // +kubebuilder:validation:XValidation:message="windows families may only specify version 'latest'",rule="self.split('@')[0] in ['windows2019','windows2022'] ? self.split('@')[1] == 'latest' : true" + // +kubebuilder:validation:MaxLength=30 + // +optional + Alias string `json:"alias,omitempty"` + // Tags is a map of key/value tags used to select subnets + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the ami id in EC2 + // +kubebuilder:validation:Pattern:="ami-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` + // Name is the ami name in EC2. + // This value is the name field, which is different from the name tag. + // +optional + Name string `json:"name,omitempty"` + // Owner is the owner for the ami. + // You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + // +optional + Owner string `json:"owner,omitempty"` +} + +// KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes. +// They are a subset of the upstream types, recognizing not all options may be supported. +// Wherever possible, the types and names should reflect the upstream kubelet types. +// https://pkg.go.dev/k8s.io/kubelet/config/v1beta1#KubeletConfiguration +// https://github.com/kubernetes/kubernetes/blob/9f82d81e55cafdedab619ea25cabf5d42736dacf/cmd/kubelet/app/options/options.go#L53 +type KubeletConfiguration struct { + // clusterDNS is a list of IP addresses for the cluster DNS server. + // Note that not all providers may use all addresses. + //+optional + ClusterDNS []string `json:"clusterDNS,omitempty"` + // MaxPods is an override for the maximum number of pods that can run on + // a worker node instance. + // +kubebuilder:validation:Minimum:=0 + // +optional + MaxPods *int32 `json:"maxPods,omitempty"` + // PodsPerCore is an override for the number of pods that can run on a worker node + // instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + // MaxPods is a lower value, that value will be used. + // +kubebuilder:validation:Minimum:=0 + // +optional + PodsPerCore *int32 `json:"podsPerCore,omitempty"` + // SystemReserved contains resources reserved for OS system daemons and kernel memory. + // +kubebuilder:validation:XValidation:message="valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')" + // +kubebuilder:validation:XValidation:message="systemReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))" + // +optional + SystemReserved map[string]string `json:"systemReserved,omitempty"` + // KubeReserved contains resources reserved for Kubernetes system components. + // +kubebuilder:validation:XValidation:message="valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')" + // +kubebuilder:validation:XValidation:message="kubeReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))" + // +optional + KubeReserved map[string]string `json:"kubeReserved,omitempty"` + // EvictionHard is the map of signal names to quantities that define hard eviction thresholds + // +kubebuilder:validation:XValidation:message="valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + // +optional + EvictionHard map[string]string `json:"evictionHard,omitempty"` + // EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + // +kubebuilder:validation:XValidation:message="valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + // +optional + EvictionSoft map[string]string `json:"evictionSoft,omitempty"` + // EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + // +kubebuilder:validation:XValidation:message="valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + // +optional + EvictionSoftGracePeriod map[string]metav1.Duration `json:"evictionSoftGracePeriod,omitempty"` + // EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + // response to soft eviction thresholds being met. + // +optional + EvictionMaxPodGracePeriod *int32 `json:"evictionMaxPodGracePeriod,omitempty"` + // ImageGCHighThresholdPercent is the percent of disk usage after which image + // garbage collection is always run. The percent is calculated by dividing this + // field value by 100, so this field must be between 0 and 100, inclusive. + // When specified, the value must be greater than ImageGCLowThresholdPercent. + // +kubebuilder:validation:Minimum:=0 + // +kubebuilder:validation:Maximum:=100 + // +optional + ImageGCHighThresholdPercent *int32 `json:"imageGCHighThresholdPercent,omitempty"` + // ImageGCLowThresholdPercent is the percent of disk usage before which image + // garbage collection is never run. Lowest disk usage to garbage collect to. + // The percent is calculated by dividing this field value by 100, + // so the field value must be between 0 and 100, inclusive. + // When specified, the value must be less than imageGCHighThresholdPercent + // +kubebuilder:validation:Minimum:=0 + // +kubebuilder:validation:Maximum:=100 + // +optional + ImageGCLowThresholdPercent *int32 `json:"imageGCLowThresholdPercent,omitempty"` + // CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + // +optional + CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"` +} + +// MetadataOptions contains parameters for specifying the exposure of the +// Instance Metadata Service to provisioned EC2 nodes. +type MetadataOptions struct { + // HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + // nodes. If metadata options is non-nil, but this parameter is not specified, + // the default state is "enabled". + // + // If you specify a value of "disabled", instance metadata will not be accessible + // on the node. + // +kubebuilder:default=enabled + // +kubebuilder:validation:Enum:={enabled,disabled} + // +optional + HTTPEndpoint *string `json:"httpEndpoint,omitempty"` + // HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + // service on provisioned nodes. If metadata options is non-nil, but this parameter + // is not specified, the default state is "disabled". + // +kubebuilder:default=disabled + // +kubebuilder:validation:Enum:={enabled,disabled} + // +optional + HTTPProtocolIPv6 *string `json:"httpProtocolIPv6,omitempty"` + // HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + // instance metadata requests. The larger the number, the further instance + // metadata requests can travel. Possible values are integers from 1 to 64. + // If metadata options is non-nil, but this parameter is not specified, the + // default value is 1. + // +kubebuilder:default=1 + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=64 + // +optional + HTTPPutResponseHopLimit *int64 `json:"httpPutResponseHopLimit,omitempty"` + // HTTPTokens determines the state of token usage for instance metadata + // requests. If metadata options is non-nil, but this parameter is not + // specified, the default state is "required". + // + // If the state is optional, one can choose to retrieve instance metadata with + // or without a signed token header on the request. If one retrieves the IAM + // role credentials without a token, the version 1.0 role credentials are + // returned. If one retrieves the IAM role credentials using a valid signed + // token, the version 2.0 role credentials are returned. + // + // If the state is "required", one must send a signed token header with any + // instance metadata retrieval requests. In this state, retrieving the IAM + // role credentials always returns the version 2.0 credentials; the version + // 1.0 credentials are not available. + // +kubebuilder:default=required + // +kubebuilder:validation:Enum:={required,optional} + // +optional + HTTPTokens *string `json:"httpTokens,omitempty"` +} + +type BlockDeviceMapping struct { + // The device name (for example, /dev/sdh or xvdh). + // +required + DeviceName *string `json:"deviceName,omitempty"` + // EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + // +kubebuilder:validation:XValidation:message="snapshotID or volumeSize must be defined",rule="has(self.snapshotID) || has(self.volumeSize)" + // +required + EBS *BlockDevice `json:"ebs,omitempty"` + // RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + // configure at most one root volume in BlockDeviceMappings. + RootVolume bool `json:"rootVolume,omitempty"` +} + +type BlockDevice struct { + // DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + // +optional + DeleteOnTermination *bool `json:"deleteOnTermination,omitempty"` + // Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + // be attached to instances that support Amazon EBS encryption. If you are creating + // a volume from a snapshot, you can't specify an encryption value. + // +optional + Encrypted *bool `json:"encrypted,omitempty"` + // IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + // this represents the number of IOPS that are provisioned for the volume. For + // gp2 volumes, this represents the baseline performance of the volume and the + // rate at which the volume accumulates I/O credits for bursting. + // + // The following are the supported values for each volume type: + // + // * gp3: 3,000-16,000 IOPS + // + // * io1: 100-64,000 IOPS + // + // * io2: 100-64,000 IOPS + // + // For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + // on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + // Other instance families guarantee performance up to 32,000 IOPS. + // + // This parameter is supported for io1, io2, and gp3 volumes only. This parameter + // is not supported for gp2, st1, sc1, or standard volumes. + // +optional + IOPS *int64 `json:"iops,omitempty"` + // KMSKeyID (ARN) of the symmetric Key Management Service (KMS) CMK used for encryption. + // +optional + KMSKeyID *string `json:"kmsKeyID,omitempty"` + // SnapshotID is the ID of an EBS snapshot + // +optional + SnapshotID *string `json:"snapshotID,omitempty"` + // Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + // Valid Range: Minimum value of 125. Maximum value of 1000. + // +optional + Throughput *int64 `json:"throughput,omitempty"` + // VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + // a volume size. The following are the supported volumes sizes for each volume + // type: + // + // * gp2 and gp3: 1-16,384 + // + // * io1 and io2: 4-16,384 + // + // * st1 and sc1: 125-16,384 + // + // * standard: 1-1,024 + // + TODO: Add the CEL resources.quantity type after k8s 1.29 + // + https://github.com/kubernetes/apiserver/commit/b137c256373aec1c5d5810afbabb8932a19ecd2a#diff-838176caa5882465c9d6061febd456397a3e2b40fb423ed36f0cabb1847ecb4dR190 + // +kubebuilder:validation:Pattern:="^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$" + // +kubebuilder:validation:Schemaless + // +kubebuilder:validation:Type:=string + // +optional + VolumeSize *resource.Quantity `json:"volumeSize,omitempty" hash:"string"` + // VolumeType of the block device. + // For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + // in the Amazon Elastic Compute Cloud User Guide. + // +kubebuilder:validation:Enum:={standard,io1,io2,gp2,sc1,st1,gp3} + // +optional + VolumeType *string `json:"volumeType,omitempty"` +} + +// InstanceStorePolicy enumerates options for configuring instance store disks. +// +kubebuilder:validation:Enum={RAID0} +type InstanceStorePolicy string + +const ( + // InstanceStorePolicyRAID0 configures a RAID-0 array that includes all ephemeral NVMe instance storage disks. + // The containerd and kubelet state directories (`/var/lib/containerd` and `/var/lib/kubelet`) will then use the + // ephemeral storage for more and faster node ephemeral-storage. The node's ephemeral storage can be shared among + // pods that request ephemeral storage and container images that are downloaded to the node. + InstanceStorePolicyRAID0 InstanceStorePolicy = "RAID0" +) + +// EC2NodeClass is the Schema for the EC2NodeClass API +// +kubebuilder:object:root=true +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="" +// +kubebuilder:printcolumn:name="Role",type="string",JSONPath=".spec.role",priority=1,description="" +// +kubebuilder:resource:path=ec2nodeclasses,scope=Cluster,categories=karpenter,shortName={ec2nc,ec2ncs} +// +kubebuilder:storageversion +// +kubebuilder:subresource:status +type EC2NodeClass struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // +kubebuilder:validation:XValidation:message="must specify exactly one of ['role', 'instanceProfile']",rule="(has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile))" + // +kubebuilder:validation:XValidation:message="changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this.",rule="(has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile))" + // +kubebuilder:validation:XValidation:message="if set, amiFamily must be 'AL2' or 'Custom' when using an AL2 alias",rule="!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find('^[^@]+') == 'al2') ? (self.amiFamily == 'Custom' || self.amiFamily == 'AL2') : true)" + // +kubebuilder:validation:XValidation:message="if set, amiFamily must be 'AL2023' or 'Custom' when using an AL2023 alias",rule="!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find('^[^@]+') == 'al2023') ? (self.amiFamily == 'Custom' || self.amiFamily == 'AL2023') : true)" + // +kubebuilder:validation:XValidation:message="if set, amiFamily must be 'Bottlerocket' or 'Custom' when using a Bottlerocket alias",rule="!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find('^[^@]+') == 'bottlerocket') ? (self.amiFamily == 'Custom' || self.amiFamily == 'Bottlerocket') : true)" + // +kubebuilder:validation:XValidation:message="if set, amiFamily must be 'Windows2019' or 'Custom' when using a Windows2019 alias",rule="!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find('^[^@]+') == 'windows2019') ? (self.amiFamily == 'Custom' || self.amiFamily == 'Windows2019') : true)" + // +kubebuilder:validation:XValidation:message="if set, amiFamily must be 'Windows2022' or 'Custom' when using a Windows2022 alias",rule="!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find('^[^@]+') == 'windows2022') ? (self.amiFamily == 'Custom' || self.amiFamily == 'Windows2022') : true)" + // +kubebuilder:validation:XValidation:message="must specify amiFamily if amiSelectorTerms does not contain an alias",rule="self.amiSelectorTerms.exists(x, has(x.alias)) ? true : has(self.amiFamily)" + Spec EC2NodeClassSpec `json:"spec,omitempty"` + Status EC2NodeClassStatus `json:"status,omitempty"` +} + +// We need to bump the EC2NodeClassHashVersion when we make an update to the EC2NodeClass CRD under these conditions: +// 1. A field changes its default value for an existing field that is already hashed +// 2. A field is added to the hash calculation with an already-set value +// 3. A field is removed from the hash calculations +const EC2NodeClassHashVersion = "v3" + +func (in *EC2NodeClass) Hash() string { + return fmt.Sprint(lo.Must(hashstructure.Hash([]interface{}{ + in.Spec, + // AMIFamily should be hashed using the dynamically resolved value rather than the literal value of the field. + // This ensures that scenarios such as changing the field from nil to AL2023 with the alias "al2023@latest" + // doesn't trigger drift. + in.AMIFamily(), + }, hashstructure.FormatV2, &hashstructure.HashOptions{ + SlicesAsSets: true, + IgnoreZeroValue: true, + ZeroNil: true, + }))) +} + +func (in *EC2NodeClass) InstanceProfileName(clusterName, region string) string { + return fmt.Sprintf("%s_%d", clusterName, lo.Must(hashstructure.Hash(fmt.Sprintf("%s%s", region, in.Name), hashstructure.FormatV2, nil))) +} + +func (in *EC2NodeClass) InstanceProfileRole() string { + return in.Spec.Role +} + +func (in *EC2NodeClass) InstanceProfileTags(clusterName string) map[string]string { + return lo.Assign(in.Spec.Tags, map[string]string{ + fmt.Sprintf("kubernetes.io/cluster/%s", clusterName): "owned", + EKSClusterNameTagKey: clusterName, + LabelNodeClass: in.Name, + }) +} + +// UbuntuIncompatible returns true if the NodeClass has the ubuntu compatibility annotation. This will cause the NodeClass to show +// as NotReady in its status conditions, opting its referencing NodePools out of provisioning and drift. +func (in *EC2NodeClass) UbuntuIncompatible() bool { + return lo.Contains(strings.Split(in.Annotations[AnnotationUbuntuCompatibilityKey], ","), AnnotationUbuntuCompatibilityIncompatible) +} + +// AMIFamily returns the family for a NodePool based on the following items, in order of precdence: +// - ec2nodeclass.spec.amiFamily +// - ec2nodeclass.spec.amiSelectorTerms[].alias +// +// If an alias is specified, ec2nodeclass.spec.amiFamily must match that alias, or be 'Custom' (enforced via validation). +func (in *EC2NodeClass) AMIFamily() string { + if in.Spec.AMIFamily != nil { + return *in.Spec.AMIFamily + } + if term, ok := lo.Find(in.Spec.AMISelectorTerms, func(t AMISelectorTerm) bool { + return t.Alias != "" + }); ok { + return AMIFamilyFromAlias(term.Alias) + } + // Unreachable: validation enforces that one of the above conditions must be met + return AMIFamilyCustom +} + +func AMIFamilyFromAlias(alias string) string { + components := strings.Split(alias, "@") + if len(components) != 2 { + log.Fatalf("failed to parse AMI alias %q, invalid format", alias) + } + family, ok := lo.Find([]string{ + AMIFamilyAL2, + AMIFamilyAL2023, + AMIFamilyBottlerocket, + AMIFamilyWindows2019, + AMIFamilyWindows2022, + }, func(family string) bool { + return strings.ToLower(family) == components[0] + }) + if !ok { + log.Fatalf("%q is an invalid alias family", components[0]) + } + return family +} + +func AMIVersionFromAlias(alias string) string { + components := strings.Split(alias, "@") + if len(components) != 2 { + log.Fatalf("failed to parse AMI alias %q, invalid format", alias) + } + return components[1] +} + +// EC2NodeClassList contains a list of EC2NodeClass +// +kubebuilder:object:root=true +type EC2NodeClassList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EC2NodeClass `json:"items"` +} diff --git a/pkg/apis/v1/ec2nodeclass_conversion.go b/pkg/apis/v1/ec2nodeclass_conversion.go new file mode 100644 index 000000000000..caa5284d13c3 --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass_conversion.go @@ -0,0 +1,256 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "fmt" + "strings" + + "github.com/samber/lo" + "k8s.io/apimachinery/pkg/api/resource" + "knative.dev/pkg/apis" + + "github.com/aws/aws-sdk-go/service/ec2" + + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" +) + +func (in *EC2NodeClass) ConvertTo(ctx context.Context, to apis.Convertible) error { + v1beta1enc := to.(*v1beta1.EC2NodeClass) + v1beta1enc.ObjectMeta = in.ObjectMeta + v1beta1enc.Annotations = lo.OmitByKeys(v1beta1enc.Annotations, []string{AnnotationUbuntuCompatibilityKey}) + + if value, ok := in.Annotations[AnnotationUbuntuCompatibilityKey]; ok { + compatSpecifiers := strings.Split(value, ",") + // Remove the `id: ami-placeholder` AMISelectorTerms that are injected to pass CRD validation at v1 + // we don't need these in v1beta1, and should be dropped + if lo.Contains(compatSpecifiers, AnnotationUbuntuCompatibilityIncompatible) { + in.Spec.AMISelectorTerms = nil + } + // The only blockDeviceMappings present on the v1 EC2NodeClass are those that we injected during conversion. + // These should be dropped. + if lo.Contains(compatSpecifiers, AnnotationUbuntuCompatibilityBlockDeviceMappings) { + in.Spec.BlockDeviceMappings = nil + } + // We don't need to explicitly check for the AMIFamily compat specifier, the presence of the annotation implies its existence + v1beta1enc.Spec.AMIFamily = lo.ToPtr(AMIFamilyUbuntu) + } else { + v1beta1enc.Spec.AMIFamily = lo.ToPtr(in.AMIFamily()) + } + + in.Spec.convertTo(&v1beta1enc.Spec) + in.Status.convertTo((&v1beta1enc.Status)) + return nil +} + +func (in *EC2NodeClassSpec) convertTo(v1beta1enc *v1beta1.EC2NodeClassSpec) { + v1beta1enc.SubnetSelectorTerms = lo.Map(in.SubnetSelectorTerms, func(subnet SubnetSelectorTerm, _ int) v1beta1.SubnetSelectorTerm { + return v1beta1.SubnetSelectorTerm{ + ID: subnet.ID, + Tags: subnet.Tags, + } + }) + v1beta1enc.SecurityGroupSelectorTerms = lo.Map(in.SecurityGroupSelectorTerms, func(sg SecurityGroupSelectorTerm, _ int) v1beta1.SecurityGroupSelectorTerm { + return v1beta1.SecurityGroupSelectorTerm{ + ID: sg.ID, + Name: sg.Name, + Tags: sg.Tags, + } + }) + v1beta1enc.AMISelectorTerms = lo.FilterMap(in.AMISelectorTerms, func(term AMISelectorTerm, _ int) (v1beta1.AMISelectorTerm, bool) { + if term.Alias != "" { + return v1beta1.AMISelectorTerm{}, false + } + return v1beta1.AMISelectorTerm{ + ID: term.ID, + Name: term.Name, + Owner: term.Owner, + Tags: term.Tags, + }, true + }) + v1beta1enc.AssociatePublicIPAddress = in.AssociatePublicIPAddress + v1beta1enc.Context = in.Context + v1beta1enc.DetailedMonitoring = in.DetailedMonitoring + v1beta1enc.Role = in.Role + v1beta1enc.InstanceProfile = in.InstanceProfile + v1beta1enc.InstanceStorePolicy = (*v1beta1.InstanceStorePolicy)(in.InstanceStorePolicy) + v1beta1enc.Tags = in.Tags + v1beta1enc.UserData = in.UserData + v1beta1enc.MetadataOptions = (*v1beta1.MetadataOptions)(in.MetadataOptions) + v1beta1enc.BlockDeviceMappings = lo.Map(in.BlockDeviceMappings, func(bdm *BlockDeviceMapping, _ int) *v1beta1.BlockDeviceMapping { + return &v1beta1.BlockDeviceMapping{ + DeviceName: bdm.DeviceName, + RootVolume: bdm.RootVolume, + EBS: (*v1beta1.BlockDevice)(bdm.EBS), + } + }) +} + +func (in *EC2NodeClassStatus) convertTo(v1beta1enc *v1beta1.EC2NodeClassStatus) { + v1beta1enc.Subnets = lo.Map(in.Subnets, func(subnet Subnet, _ int) v1beta1.Subnet { + return v1beta1.Subnet{ + ID: subnet.ID, + Zone: subnet.Zone, + ZoneID: subnet.ZoneID, + } + }) + v1beta1enc.SecurityGroups = lo.Map(in.SecurityGroups, func(sg SecurityGroup, _ int) v1beta1.SecurityGroup { + return v1beta1.SecurityGroup{ + ID: sg.ID, + Name: sg.Name, + } + }) + v1beta1enc.AMIs = lo.Map(in.AMIs, func(ami AMI, _ int) v1beta1.AMI { + return v1beta1.AMI{ + ID: ami.ID, + Name: ami.Name, + Requirements: ami.Requirements, + } + }) + v1beta1enc.InstanceProfile = in.InstanceProfile + v1beta1enc.Conditions = in.Conditions +} + +func (in *EC2NodeClass) ConvertFrom(ctx context.Context, from apis.Convertible) error { + v1beta1enc := from.(*v1beta1.EC2NodeClass) + in.ObjectMeta = v1beta1enc.ObjectMeta + + switch lo.FromPtr(v1beta1enc.Spec.AMIFamily) { + case AMIFamilyAL2, AMIFamilyAL2023, AMIFamilyBottlerocket, AMIFamilyWindows2019, AMIFamilyWindows2022: + // If no amiSelectorTerms are specified, we can create an alias and don't need to specify amiFamily. Otherwise, + // we'll carry over the amiSelectorTerms and amiFamily. + if len(v1beta1enc.Spec.AMISelectorTerms) == 0 { + in.Spec.AMIFamily = nil + in.Spec.AMISelectorTerms = []AMISelectorTerm{{ + Alias: fmt.Sprintf("%s@latest", strings.ToLower(lo.FromPtr(v1beta1enc.Spec.AMIFamily))), + }} + } else { + in.Spec.AMIFamily = v1beta1enc.Spec.AMIFamily + } + case AMIFamilyUbuntu: + // If AMISelectorTerms were specified, we can continue to use them to discover Ubuntu AMIs and use the AL2 AMI + // family for bootstrapping. AL2 and Ubuntu have an identical UserData format, but do have different default + // BlockDeviceMappings. We'll set the BlockDeviceMappings to Ubuntu's default if no user specified + // BlockDeviceMappings are present. + compatSpecifiers := []string{AnnotationUbuntuCompatibilityAMIFamily} + in.Spec.AMIFamily = lo.ToPtr(AMIFamilyAL2) + if v1beta1enc.Spec.BlockDeviceMappings == nil { + compatSpecifiers = append(compatSpecifiers, AnnotationUbuntuCompatibilityBlockDeviceMappings) + in.Spec.BlockDeviceMappings = []*BlockDeviceMapping{{ + DeviceName: lo.ToPtr("/dev/sda1"), + RootVolume: true, + EBS: &BlockDevice{ + Encrypted: lo.ToPtr(true), + VolumeType: lo.ToPtr(ec2.VolumeTypeGp3), + VolumeSize: lo.ToPtr(resource.MustParse("20Gi")), + }, + }} + } + + // If there are no AMISelectorTerms specified, we mark the ec2nodeclass as incompatible. + // Karpenter will ignore incompatible ec2nodeclasses for provisioning and computing drift. + if len(v1beta1enc.Spec.AMISelectorTerms) == 0 { + compatSpecifiers = append(compatSpecifiers, AnnotationUbuntuCompatibilityIncompatible) + in.Spec.AMISelectorTerms = []AMISelectorTerm{{ + ID: "ami-placeholder", + }} + } + + // This compatibility annotation will be used to determine if the amiFamily was mutated from Ubuntu to AL2, and + // if we needed to inject any blockDeviceMappings. This is required to enable a round-trip conversion. + in.Annotations = lo.Assign(in.Annotations, map[string]string{ + AnnotationUbuntuCompatibilityKey: strings.Join(compatSpecifiers, ","), + }) + default: + // The amiFamily is custom or undefined (shouldn't be possible via validation). We'll treat it as custom + // regardless. + in.Spec.AMIFamily = lo.ToPtr(AMIFamilyCustom) + } + + in.Spec.convertFrom(&v1beta1enc.Spec) + in.Status.convertFrom(&v1beta1enc.Status) + return nil +} + +func (in *EC2NodeClassSpec) convertFrom(v1beta1enc *v1beta1.EC2NodeClassSpec) { + if in.AMISelectorTerms == nil { + in.AMISelectorTerms = lo.Map(v1beta1enc.AMISelectorTerms, func(ami v1beta1.AMISelectorTerm, _ int) AMISelectorTerm { + return AMISelectorTerm{ + ID: ami.ID, + Name: ami.Name, + Owner: ami.Owner, + Tags: ami.Tags, + } + }) + } + if in.BlockDeviceMappings == nil { + in.BlockDeviceMappings = lo.Map(v1beta1enc.BlockDeviceMappings, func(bdm *v1beta1.BlockDeviceMapping, _ int) *BlockDeviceMapping { + return &BlockDeviceMapping{ + DeviceName: bdm.DeviceName, + RootVolume: bdm.RootVolume, + EBS: (*BlockDevice)(bdm.EBS), + } + }) + } + + in.SubnetSelectorTerms = lo.Map(v1beta1enc.SubnetSelectorTerms, func(subnet v1beta1.SubnetSelectorTerm, _ int) SubnetSelectorTerm { + return SubnetSelectorTerm{ + ID: subnet.ID, + Tags: subnet.Tags, + } + }) + in.SecurityGroupSelectorTerms = lo.Map(v1beta1enc.SecurityGroupSelectorTerms, func(sg v1beta1.SecurityGroupSelectorTerm, _ int) SecurityGroupSelectorTerm { + return SecurityGroupSelectorTerm{ + ID: sg.ID, + Name: sg.Name, + Tags: sg.Tags, + } + }) + in.AssociatePublicIPAddress = v1beta1enc.AssociatePublicIPAddress + in.Context = v1beta1enc.Context + in.DetailedMonitoring = v1beta1enc.DetailedMonitoring + in.Role = v1beta1enc.Role + in.InstanceProfile = v1beta1enc.InstanceProfile + in.InstanceStorePolicy = (*InstanceStorePolicy)(v1beta1enc.InstanceStorePolicy) + in.Tags = v1beta1enc.Tags + in.UserData = v1beta1enc.UserData + in.MetadataOptions = (*MetadataOptions)(v1beta1enc.MetadataOptions) +} + +func (in *EC2NodeClassStatus) convertFrom(v1beta1enc *v1beta1.EC2NodeClassStatus) { + in.Subnets = lo.Map(v1beta1enc.Subnets, func(subnet v1beta1.Subnet, _ int) Subnet { + return Subnet{ + ID: subnet.ID, + Zone: subnet.Zone, + ZoneID: subnet.ZoneID, + } + }) + in.SecurityGroups = lo.Map(v1beta1enc.SecurityGroups, func(sg v1beta1.SecurityGroup, _ int) SecurityGroup { + return SecurityGroup{ + ID: sg.ID, + Name: sg.Name, + } + }) + in.AMIs = lo.Map(v1beta1enc.AMIs, func(ami v1beta1.AMI, _ int) AMI { + return AMI{ + ID: ami.ID, + Name: ami.Name, + Requirements: ami.Requirements, + } + }) + in.InstanceProfile = v1beta1enc.InstanceProfile + in.Conditions = v1beta1enc.Conditions +} diff --git a/pkg/apis/v1/ec2nodeclass_conversion_test.go b/pkg/apis/v1/ec2nodeclass_conversion_test.go new file mode 100644 index 000000000000..4c25eec22ca7 --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass_conversion_test.go @@ -0,0 +1,610 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1_test + +import ( + "fmt" + + "github.com/awslabs/operatorpkg/status" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/karpenter/pkg/test" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/aws/aws-sdk-go/service/ec2" + + . "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" +) + +var _ = Describe("Convert v1 to v1beta1 EC2NodeClass API", func() { + var ( + v1ec2nodeclass *EC2NodeClass + v1beta1ec2nodeclass *v1beta1.EC2NodeClass + ) + + BeforeEach(func() { + v1ec2nodeclass = &EC2NodeClass{} + v1beta1ec2nodeclass = &v1beta1.EC2NodeClass{} + }) + + It("should convert v1 ec2nodeclass metadata", func() { + v1ec2nodeclass.ObjectMeta = test.ObjectMeta(metav1.ObjectMeta{ + Annotations: map[string]string{"foo": "bar"}, + }) + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1beta1ec2nodeclass.ObjectMeta).To(BeEquivalentTo(v1ec2nodeclass.ObjectMeta)) + }) + Context("EC2NodeClass Spec", func() { + It("should convert v1 ec2nodeclass subnet selector terms", func() { + v1ec2nodeclass.Spec.SubnetSelectorTerms = []SubnetSelectorTerm{ + { + Tags: map[string]string{"test-key-1": "test-value-1"}, + ID: "test-id-1", + }, + { + Tags: map[string]string{"test-key-2": "test-value-2"}, + ID: "test-id-2", + }, + } + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1ec2nodeclass.Spec.SubnetSelectorTerms { + Expect(v1beta1ec2nodeclass.Spec.SubnetSelectorTerms[i].Tags).To(Equal(v1ec2nodeclass.Spec.SubnetSelectorTerms[i].Tags)) + Expect(v1beta1ec2nodeclass.Spec.SubnetSelectorTerms[i].ID).To(Equal(v1ec2nodeclass.Spec.SubnetSelectorTerms[i].ID)) + } + }) + It("should convert v1 ec2nodeclass securitygroup selector terms", func() { + v1ec2nodeclass.Spec.SecurityGroupSelectorTerms = []SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"test-key-1": "test-value-1"}, + ID: "test-id-1", + Name: "test-name-1", + }, + { + Tags: map[string]string{"test-key-2": "test-value-2"}, + ID: "test-id-2", + Name: "test-name-2", + }, + } + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1ec2nodeclass.Spec.SecurityGroupSelectorTerms { + Expect(v1beta1ec2nodeclass.Spec.SecurityGroupSelectorTerms[i].Tags).To(Equal(v1ec2nodeclass.Spec.SecurityGroupSelectorTerms[i].Tags)) + Expect(v1beta1ec2nodeclass.Spec.SecurityGroupSelectorTerms[i].ID).To(Equal(v1ec2nodeclass.Spec.SecurityGroupSelectorTerms[i].ID)) + Expect(v1beta1ec2nodeclass.Spec.SecurityGroupSelectorTerms[i].Name).To(Equal(v1ec2nodeclass.Spec.SecurityGroupSelectorTerms[i].Name)) + } + }) + It("should convert v1 ec2nodeclass ami selector terms", func() { + v1ec2nodeclass.Spec.AMISelectorTerms = []AMISelectorTerm{ + { + Tags: map[string]string{"test-key-1": "test-value-1"}, + ID: "test-id-1", + Name: "test-name-1", + Owner: "test-owner-1", + }, + { + Tags: map[string]string{"test-key-2": "test-value-2"}, + ID: "test-id-2", + Name: "test-name-2", + Owner: "test-owner-1", + }, + } + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1ec2nodeclass.Spec.AMISelectorTerms { + Expect(v1beta1ec2nodeclass.Spec.AMISelectorTerms[i].Tags).To(Equal(v1ec2nodeclass.Spec.AMISelectorTerms[i].Tags)) + Expect(v1beta1ec2nodeclass.Spec.AMISelectorTerms[i].ID).To(Equal(v1ec2nodeclass.Spec.AMISelectorTerms[i].ID)) + Expect(v1beta1ec2nodeclass.Spec.AMISelectorTerms[i].Name).To(Equal(v1ec2nodeclass.Spec.AMISelectorTerms[i].Name)) + Expect(v1beta1ec2nodeclass.Spec.AMISelectorTerms[i].Owner).To(Equal(v1ec2nodeclass.Spec.AMISelectorTerms[i].Owner)) + } + }) + It("should convert v1 ec2nodeclass associate public ip address ", func() { + v1ec2nodeclass.Spec.AssociatePublicIPAddress = lo.ToPtr(true) + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.AssociatePublicIPAddress)).To(BeTrue()) + }) + It("should convert v1 ec2nodeclass alias", func() { + v1ec2nodeclass.Spec.AMISelectorTerms = []AMISelectorTerm{{Alias: "al2023@latest"}} + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.AMIFamily)).To(Equal(v1beta1.AMIFamilyAL2023)) + }) + It("should convert v1 ec2nodeclass ami selector terms with the Ubuntu compatibility annotation", func() { + v1ec2nodeclass.Annotations = lo.Assign(v1ec2nodeclass.Annotations, map[string]string{ + AnnotationUbuntuCompatibilityKey: fmt.Sprintf("%s,%s", AnnotationUbuntuCompatibilityAMIFamily, AnnotationUbuntuCompatibilityBlockDeviceMappings), + }) + v1ec2nodeclass.Spec.AMIFamily = lo.ToPtr(AMIFamilyAL2) + v1ec2nodeclass.Spec.AMISelectorTerms = []AMISelectorTerm{{ID: "ami-01234567890abcdef"}} + v1ec2nodeclass.Spec.BlockDeviceMappings = []*BlockDeviceMapping{{ + DeviceName: lo.ToPtr("/dev/sda1"), + RootVolume: true, + EBS: &BlockDevice{ + Encrypted: lo.ToPtr(true), + VolumeType: lo.ToPtr(ec2.VolumeTypeGp3), + VolumeSize: lo.ToPtr(resource.MustParse("20Gi")), + }, + }} + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1beta1ec2nodeclass.Annotations).ToNot(HaveKey(AnnotationUbuntuCompatibilityKey)) + Expect(len(v1beta1ec2nodeclass.Spec.BlockDeviceMappings)).To(Equal(0)) + Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.AMIFamily)).To(Equal(v1beta1.AMIFamilyUbuntu)) + Expect(v1beta1ec2nodeclass.Spec.AMISelectorTerms).To(Equal([]v1beta1.AMISelectorTerm{{ID: "ami-01234567890abcdef"}})) + }) + It("should convert v1 ec2nodeclass ami selector terms with the Ubuntu compatibility annotation and custom BlockDeviceMappings", func() { + v1ec2nodeclass.Annotations = lo.Assign(v1ec2nodeclass.Annotations, map[string]string{AnnotationUbuntuCompatibilityKey: AnnotationUbuntuCompatibilityAMIFamily}) + v1ec2nodeclass.Spec.AMIFamily = lo.ToPtr(AMIFamilyAL2) + v1ec2nodeclass.Spec.AMISelectorTerms = []AMISelectorTerm{{ID: "ami-01234567890abcdef"}} + v1ec2nodeclass.Spec.BlockDeviceMappings = []*BlockDeviceMapping{{ + DeviceName: lo.ToPtr("/dev/sdb1"), + RootVolume: true, + EBS: &BlockDevice{ + Encrypted: lo.ToPtr(false), + VolumeType: lo.ToPtr(ec2.VolumeTypeGp2), + VolumeSize: lo.ToPtr(resource.MustParse("40Gi")), + }, + }} + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1beta1ec2nodeclass.Annotations).ToNot(HaveKey(AnnotationUbuntuCompatibilityKey)) + Expect(v1beta1ec2nodeclass.Spec.BlockDeviceMappings).To(Equal([]*v1beta1.BlockDeviceMapping{{ + DeviceName: lo.ToPtr("/dev/sdb1"), + RootVolume: true, + EBS: &v1beta1.BlockDevice{ + Encrypted: lo.ToPtr(false), + VolumeType: lo.ToPtr(ec2.VolumeTypeGp2), + VolumeSize: lo.ToPtr(resource.MustParse("40Gi")), + }, + }})) + Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.AMIFamily)).To(Equal(v1beta1.AMIFamilyUbuntu)) + Expect(v1beta1ec2nodeclass.Spec.AMISelectorTerms).To(Equal([]v1beta1.AMISelectorTerm{{ID: "ami-01234567890abcdef"}})) + }) + It("should convert v1 ec2nodeclass user data", func() { + v1ec2nodeclass.Spec.UserData = lo.ToPtr("test user data") + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.UserData)).To(Equal(lo.FromPtr(v1ec2nodeclass.Spec.UserData))) + }) + It("should convert v1 ec2nodeclass role", func() { + v1ec2nodeclass.Spec.Role = "test-role" + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1beta1ec2nodeclass.Spec.Role).To(Equal(v1ec2nodeclass.Spec.Role)) + }) + It("should convert v1 ec2nodeclass instance profile", func() { + v1ec2nodeclass.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.InstanceProfile)).To(Equal(lo.FromPtr(v1ec2nodeclass.Spec.InstanceProfile))) + }) + It("should convert v1 ec2nodeclass tags", func() { + v1ec2nodeclass.Spec.Tags = map[string]string{ + "test-key-tag-1": "test-value-tag-1", + "test-key-tag-2": "test-value-tag-2", + } + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1beta1ec2nodeclass.Spec.Tags).To(Equal(v1ec2nodeclass.Spec.Tags)) + }) + It("should convert v1 ec2nodeclass block device mapping", func() { + v1ec2nodeclass.Spec.BlockDeviceMappings = []*BlockDeviceMapping{ + { + EBS: &BlockDevice{ + DeleteOnTermination: lo.ToPtr(true), + Encrypted: lo.ToPtr(true), + IOPS: lo.ToPtr(int64(45123)), + KMSKeyID: lo.ToPtr("test-kms-id"), + SnapshotID: lo.ToPtr("test-snapshot-id"), + Throughput: lo.ToPtr(int64(4512433)), + VolumeSize: lo.ToPtr(resource.MustParse("54G")), + VolumeType: lo.ToPtr("test-type"), + }, + DeviceName: lo.ToPtr("test-device"), + RootVolume: true, + }, + } + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1ec2nodeclass.Spec.BlockDeviceMappings { + Expect(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].RootVolume).To(Equal(v1ec2nodeclass.Spec.BlockDeviceMappings[i].RootVolume)) + Expect(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].DeviceName).To(Equal(v1ec2nodeclass.Spec.BlockDeviceMappings[i].DeviceName)) + Expect(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.DeleteOnTermination).To(Equal(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.DeleteOnTermination)) + Expect(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.Encrypted).To(Equal(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.Encrypted)) + Expect(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.IOPS).To(Equal(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.IOPS)) + Expect(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.KMSKeyID).To(Equal(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.KMSKeyID)) + Expect(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.SnapshotID).To(Equal(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.SnapshotID)) + Expect(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.Throughput).To(Equal(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.Throughput)) + Expect(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.VolumeSize).To(Equal(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.VolumeSize)) + Expect(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.VolumeType).To(Equal(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.VolumeType)) + } + }) + It("should convert v1 ec2nodeclass instance store policy", func() { + v1ec2nodeclass.Spec.InstanceStorePolicy = lo.ToPtr(InstanceStorePolicyRAID0) + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(string(lo.FromPtr(v1beta1ec2nodeclass.Spec.InstanceStorePolicy))).To(Equal(string(lo.FromPtr(v1ec2nodeclass.Spec.InstanceStorePolicy)))) + }) + It("should convert v1 ec2nodeclass detailed monitoring", func() { + v1ec2nodeclass.Spec.DetailedMonitoring = lo.ToPtr(true) + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.DetailedMonitoring)).To(Equal(lo.FromPtr(v1ec2nodeclass.Spec.DetailedMonitoring))) + }) + It("should convert v1 ec2nodeclass metadata options", func() { + v1ec2nodeclass.Spec.MetadataOptions = &MetadataOptions{ + HTTPEndpoint: lo.ToPtr("test-endpoint"), + HTTPProtocolIPv6: lo.ToPtr("test-protocol"), + HTTPPutResponseHopLimit: lo.ToPtr(int64(54)), + HTTPTokens: lo.ToPtr("test-token"), + } + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.MetadataOptions.HTTPEndpoint)).To(Equal(lo.FromPtr(v1ec2nodeclass.Spec.MetadataOptions.HTTPEndpoint))) + Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.MetadataOptions.HTTPProtocolIPv6)).To(Equal(lo.FromPtr(v1ec2nodeclass.Spec.MetadataOptions.HTTPProtocolIPv6))) + Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.MetadataOptions.HTTPPutResponseHopLimit)).To(Equal(lo.FromPtr(v1ec2nodeclass.Spec.MetadataOptions.HTTPPutResponseHopLimit))) + Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.MetadataOptions.HTTPTokens)).To(Equal(lo.FromPtr(v1ec2nodeclass.Spec.MetadataOptions.HTTPTokens))) + }) + It("should convert v1 ec2nodeclass context", func() { + v1ec2nodeclass.Spec.Context = lo.ToPtr("test-context") + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.Context)).To(Equal(lo.FromPtr(v1ec2nodeclass.Spec.Context))) + }) + }) + Context("EC2NodeClass Status", func() { + It("should convert v1 ec2nodeclass subnet", func() { + v1ec2nodeclass.Status.Subnets = []Subnet{ + { + ID: "test-id", + Zone: "test-zone", + ZoneID: "test-zone-id", + }, + } + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1ec2nodeclass.Status.Subnets { + Expect(v1beta1ec2nodeclass.Status.Subnets[i].ID).To(Equal(v1ec2nodeclass.Status.Subnets[i].ID)) + Expect(v1beta1ec2nodeclass.Status.Subnets[i].Zone).To(Equal(v1ec2nodeclass.Status.Subnets[i].Zone)) + Expect(v1beta1ec2nodeclass.Status.Subnets[i].ZoneID).To(Equal(v1ec2nodeclass.Status.Subnets[i].ZoneID)) + } + }) + It("should convert v1 ec2nodeclass security group ", func() { + v1ec2nodeclass.Status.SecurityGroups = []SecurityGroup{ + { + ID: "test-id", + Name: "test-name", + }, + } + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1ec2nodeclass.Status.SecurityGroups { + Expect(v1beta1ec2nodeclass.Status.SecurityGroups[i].ID).To(Equal(v1ec2nodeclass.Status.SecurityGroups[i].ID)) + Expect(v1beta1ec2nodeclass.Status.SecurityGroups[i].Name).To(Equal(v1ec2nodeclass.Status.SecurityGroups[i].Name)) + } + }) + It("should convert v1 ec2nodeclass ami", func() { + v1ec2nodeclass.Status.AMIs = []AMI{ + { + ID: "test-id", + Name: "test-name", + }, + } + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1ec2nodeclass.Status.AMIs { + Expect(v1beta1ec2nodeclass.Status.AMIs[i].ID).To(Equal(v1ec2nodeclass.Status.AMIs[i].ID)) + Expect(v1beta1ec2nodeclass.Status.AMIs[i].Name).To(Equal(v1ec2nodeclass.Status.AMIs[i].Name)) + + } + }) + It("should convert v1 ec2nodeclass instance profile", func() { + v1ec2nodeclass.Status.InstanceProfile = "test-instance-profile" + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1beta1ec2nodeclass.Status.InstanceProfile).To(Equal(v1ec2nodeclass.Status.InstanceProfile)) + }) + It("should convert v1 ec2nodeclass conditions", func() { + v1ec2nodeclass.Status.Conditions = []status.Condition{ + { + Status: status.ConditionReady, + Reason: "test-reason", + }, + } + Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1beta1ec2nodeclass.Status.Conditions).To(Equal(v1beta1ec2nodeclass.Status.Conditions)) + }) + }) +}) + +var _ = Describe("Convert v1beta1 to v1 EC2NodeClass API", func() { + var ( + v1ec2nodeclass *EC2NodeClass + v1beta1ec2nodeclass *v1beta1.EC2NodeClass + ) + + BeforeEach(func() { + v1ec2nodeclass = &EC2NodeClass{} + v1beta1ec2nodeclass = &v1beta1.EC2NodeClass{} + }) + + It("should convert v1beta1 ec2nodeclass metadata", func() { + v1beta1ec2nodeclass.ObjectMeta = test.ObjectMeta(metav1.ObjectMeta{ + Annotations: map[string]string{"foo": "bar"}, + }) + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1ec2nodeclass.ObjectMeta).To(BeEquivalentTo(v1beta1ec2nodeclass.ObjectMeta)) + }) + Context("EC2NodeClass Spec", func() { + It("should convert v1beta1 ec2nodeclass subnet selector terms", func() { + v1beta1ec2nodeclass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{"test-key-1": "test-value-1"}, + ID: "test-id-1", + }, + { + Tags: map[string]string{"test-key-2": "test-value-2"}, + ID: "test-id-2", + }, + } + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1beta1ec2nodeclass.Spec.SubnetSelectorTerms { + Expect(v1ec2nodeclass.Spec.SubnetSelectorTerms[i].Tags).To(Equal(v1beta1ec2nodeclass.Spec.SubnetSelectorTerms[i].Tags)) + Expect(v1ec2nodeclass.Spec.SubnetSelectorTerms[i].ID).To(Equal(v1beta1ec2nodeclass.Spec.SubnetSelectorTerms[i].ID)) + } + }) + It("should convert v1beta1 ec2nodeclass securitygroup selector terms", func() { + v1beta1ec2nodeclass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"test-key-1": "test-value-1"}, + ID: "test-id-1", + Name: "test-name-1", + }, + { + Tags: map[string]string{"test-key-2": "test-value-2"}, + ID: "test-id-2", + Name: "test-name-2", + }, + } + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1beta1ec2nodeclass.Spec.SecurityGroupSelectorTerms { + Expect(v1ec2nodeclass.Spec.SecurityGroupSelectorTerms[i].Tags).To(Equal(v1beta1ec2nodeclass.Spec.SecurityGroupSelectorTerms[i].Tags)) + Expect(v1ec2nodeclass.Spec.SecurityGroupSelectorTerms[i].ID).To(Equal(v1beta1ec2nodeclass.Spec.SecurityGroupSelectorTerms[i].ID)) + Expect(v1ec2nodeclass.Spec.SecurityGroupSelectorTerms[i].Name).To(Equal(v1beta1ec2nodeclass.Spec.SecurityGroupSelectorTerms[i].Name)) + } + }) + It("should convert v1beta1 ec2nodeclass ami selector terms", func() { + v1beta1ec2nodeclass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Tags: map[string]string{"test-key-1": "test-value-1"}, + ID: "test-id-1", + Name: "test-name-1", + Owner: "test-owner-1", + }, + { + Tags: map[string]string{"test-key-2": "test-value-2"}, + ID: "test-id-2", + Name: "test-name-2", + Owner: "test-owner-1", + }, + } + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1beta1ec2nodeclass.Spec.AMISelectorTerms { + Expect(v1ec2nodeclass.Spec.AMISelectorTerms[i].Tags).To(Equal(v1beta1ec2nodeclass.Spec.AMISelectorTerms[i].Tags)) + Expect(v1ec2nodeclass.Spec.AMISelectorTerms[i].ID).To(Equal(v1beta1ec2nodeclass.Spec.AMISelectorTerms[i].ID)) + Expect(v1ec2nodeclass.Spec.AMISelectorTerms[i].Name).To(Equal(v1beta1ec2nodeclass.Spec.AMISelectorTerms[i].Name)) + Expect(v1ec2nodeclass.Spec.AMISelectorTerms[i].Owner).To(Equal(v1beta1ec2nodeclass.Spec.AMISelectorTerms[i].Owner)) + } + }) + It("should convert v1beta1 ec2nodeclass associate public ip address ", func() { + v1beta1ec2nodeclass.Spec.AssociatePublicIPAddress = lo.ToPtr(true) + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1ec2nodeclass.Spec.AssociatePublicIPAddress)).To(BeTrue()) + }) + It("should convert v1beta1 ec2nodeclass ami family", func() { + v1beta1ec2nodeclass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1ec2nodeclass.Spec.AMISelectorTerms).To(ContainElement(AMISelectorTerm{Alias: "al2023@latest"})) + }) + It("should convert v1beta1 ec2nodeclass ami family with non-custom ami family and ami selector terms", func() { + v1beta1ec2nodeclass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + v1beta1ec2nodeclass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + ID: "ami-0123456789abcdef", + }} + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1ec2nodeclass.Spec.AMIFamily)).To(Equal(AMIFamilyAL2023)) + Expect(v1ec2nodeclass.Spec.AMISelectorTerms).To(Equal([]AMISelectorTerm{{ + ID: "ami-0123456789abcdef", + }})) + }) + It("should convert v1beta1 ec2nodeclass when amiFamily is Ubuntu (with amiSelectorTerms)", func() { + v1beta1ec2nodeclass.Spec.AMIFamily = &v1beta1.AMIFamilyUbuntu + v1beta1ec2nodeclass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: "ami-0123456789abcdef"}} + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1ec2nodeclass.Annotations).To(HaveKeyWithValue( + AnnotationUbuntuCompatibilityKey, + fmt.Sprintf("%s,%s", AnnotationUbuntuCompatibilityAMIFamily, AnnotationUbuntuCompatibilityBlockDeviceMappings), + )) + Expect(v1ec2nodeclass.AMIFamily()).To(Equal(AMIFamilyAL2)) + Expect(v1ec2nodeclass.Spec.AMISelectorTerms).To(Equal([]AMISelectorTerm{{ID: "ami-0123456789abcdef"}})) + Expect(v1ec2nodeclass.Spec.BlockDeviceMappings).To(Equal([]*BlockDeviceMapping{{ + DeviceName: lo.ToPtr("/dev/sda1"), + RootVolume: true, + EBS: &BlockDevice{ + Encrypted: lo.ToPtr(true), + VolumeType: lo.ToPtr(ec2.VolumeTypeGp3), + VolumeSize: lo.ToPtr(resource.MustParse("20Gi")), + }, + }})) + }) + It("should convert v1beta1 ec2nodeclass when amiFamily is Ubuntu (with amiSelectorTerms and custom BlockDeviceMappings)", func() { + v1beta1ec2nodeclass.Spec.AMIFamily = &v1beta1.AMIFamilyUbuntu + v1beta1ec2nodeclass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: "ami-0123456789abcdef"}} + v1beta1ec2nodeclass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{{ + DeviceName: lo.ToPtr("/dev/sdb1"), + RootVolume: true, + EBS: &v1beta1.BlockDevice{ + Encrypted: lo.ToPtr(false), + VolumeType: lo.ToPtr(ec2.VolumeTypeGp2), + VolumeSize: lo.ToPtr(resource.MustParse("40Gi")), + }, + }} + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1ec2nodeclass.Annotations).To(HaveKeyWithValue(AnnotationUbuntuCompatibilityKey, AnnotationUbuntuCompatibilityAMIFamily)) + Expect(v1ec2nodeclass.AMIFamily()).To(Equal(AMIFamilyAL2)) + Expect(v1ec2nodeclass.Spec.AMISelectorTerms).To(Equal([]AMISelectorTerm{{ID: "ami-0123456789abcdef"}})) + Expect(v1ec2nodeclass.Spec.BlockDeviceMappings).To(Equal([]*BlockDeviceMapping{{ + DeviceName: lo.ToPtr("/dev/sdb1"), + RootVolume: true, + EBS: &BlockDevice{ + Encrypted: lo.ToPtr(false), + VolumeType: lo.ToPtr(ec2.VolumeTypeGp2), + VolumeSize: lo.ToPtr(resource.MustParse("40Gi")), + }, + }})) + }) + It("should convert v1beta1 ec2nodeclass when amiFamily is Ubuntu (without amiSelectorTerms) but mark incompatible", func() { + v1beta1ec2nodeclass.Spec.AMIFamily = lo.ToPtr(v1beta1.AMIFamilyUbuntu) + v1beta1ec2nodeclass.Spec.AMISelectorTerms = nil + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1ec2nodeclass.UbuntuIncompatible()).To(BeTrue()) + }) + It("should convert v1beta1 ec2nodeclass user data", func() { + v1beta1ec2nodeclass.Spec.UserData = lo.ToPtr("test user data") + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1ec2nodeclass.Spec.UserData)).To(Equal(lo.FromPtr(v1beta1ec2nodeclass.Spec.UserData))) + }) + It("should convert v1beta1 ec2nodeclass role", func() { + v1beta1ec2nodeclass.Spec.Role = "test-role" + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1ec2nodeclass.Spec.Role).To(Equal(v1beta1ec2nodeclass.Spec.Role)) + }) + It("should convert v1beta1 ec2nodeclass instance profile", func() { + v1beta1ec2nodeclass.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1ec2nodeclass.Spec.InstanceProfile)).To(Equal(lo.FromPtr(v1beta1ec2nodeclass.Spec.InstanceProfile))) + }) + It("should convert v1beta1 ec2nodeclass tags", func() { + v1beta1ec2nodeclass.Spec.Tags = map[string]string{ + "test-key-tag-1": "test-value-tag-1", + "test-key-tag-2": "test-value-tag-2", + } + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1ec2nodeclass.Spec.Tags).To(Equal(v1beta1ec2nodeclass.Spec.Tags)) + }) + It("should convert v1beta1 ec2nodeclass block device mapping", func() { + v1beta1ec2nodeclass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ + { + EBS: &v1beta1.BlockDevice{ + DeleteOnTermination: lo.ToPtr(true), + Encrypted: lo.ToPtr(true), + IOPS: lo.ToPtr(int64(45123)), + KMSKeyID: lo.ToPtr("test-kms-id"), + SnapshotID: lo.ToPtr("test-snapshot-id"), + Throughput: lo.ToPtr(int64(4512433)), + VolumeSize: lo.ToPtr(resource.MustParse("54G")), + VolumeType: lo.ToPtr("test-type"), + }, + DeviceName: lo.ToPtr("test-device"), + RootVolume: true, + }, + } + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1beta1ec2nodeclass.Spec.BlockDeviceMappings { + Expect(v1ec2nodeclass.Spec.BlockDeviceMappings[i].RootVolume).To(Equal(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].RootVolume)) + Expect(v1ec2nodeclass.Spec.BlockDeviceMappings[i].DeviceName).To(Equal(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].DeviceName)) + Expect(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.DeleteOnTermination).To(Equal(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.DeleteOnTermination)) + Expect(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.Encrypted).To(Equal(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.Encrypted)) + Expect(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.IOPS).To(Equal(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.IOPS)) + Expect(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.KMSKeyID).To(Equal(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.KMSKeyID)) + Expect(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.SnapshotID).To(Equal(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.SnapshotID)) + Expect(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.Throughput).To(Equal(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.Throughput)) + Expect(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.VolumeSize).To(Equal(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.VolumeSize)) + Expect(v1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.VolumeType).To(Equal(v1beta1ec2nodeclass.Spec.BlockDeviceMappings[i].EBS.VolumeType)) + } + }) + It("should convert v1beta1 ec2nodeclass instance store policy", func() { + v1beta1ec2nodeclass.Spec.InstanceStorePolicy = lo.ToPtr(v1beta1.InstanceStorePolicyRAID0) + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(string(lo.FromPtr(v1ec2nodeclass.Spec.InstanceStorePolicy))).To(Equal(string(lo.FromPtr(v1beta1ec2nodeclass.Spec.InstanceStorePolicy)))) + }) + It("should convert v1beta1 ec2nodeclass detailed monitoring", func() { + v1beta1ec2nodeclass.Spec.DetailedMonitoring = lo.ToPtr(true) + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1ec2nodeclass.Spec.DetailedMonitoring)).To(Equal(lo.FromPtr(v1beta1ec2nodeclass.Spec.DetailedMonitoring))) + }) + It("should convert v1beta1 ec2nodeclass metadata options", func() { + v1beta1ec2nodeclass.Spec.MetadataOptions = &v1beta1.MetadataOptions{ + HTTPEndpoint: lo.ToPtr("test-endpoint"), + HTTPProtocolIPv6: lo.ToPtr("test-protocol"), + HTTPPutResponseHopLimit: lo.ToPtr(int64(54)), + HTTPTokens: lo.ToPtr("test-token"), + } + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1ec2nodeclass.Spec.MetadataOptions.HTTPEndpoint)).To(Equal(lo.FromPtr(v1beta1ec2nodeclass.Spec.MetadataOptions.HTTPEndpoint))) + Expect(lo.FromPtr(v1ec2nodeclass.Spec.MetadataOptions.HTTPProtocolIPv6)).To(Equal(lo.FromPtr(v1beta1ec2nodeclass.Spec.MetadataOptions.HTTPProtocolIPv6))) + Expect(lo.FromPtr(v1ec2nodeclass.Spec.MetadataOptions.HTTPPutResponseHopLimit)).To(Equal(lo.FromPtr(v1beta1ec2nodeclass.Spec.MetadataOptions.HTTPPutResponseHopLimit))) + Expect(lo.FromPtr(v1ec2nodeclass.Spec.MetadataOptions.HTTPTokens)).To(Equal(lo.FromPtr(v1beta1ec2nodeclass.Spec.MetadataOptions.HTTPTokens))) + }) + It("should convert v1beta1 ec2nodeclass context", func() { + v1beta1ec2nodeclass.Spec.Context = lo.ToPtr("test-context") + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(lo.FromPtr(v1ec2nodeclass.Spec.Context)).To(Equal(lo.FromPtr(v1beta1ec2nodeclass.Spec.Context))) + }) + }) + Context("EC2NodeClass Status", func() { + It("should convert v1beta1 ec2nodeclass subnet", func() { + v1beta1ec2nodeclass.Status.Subnets = []v1beta1.Subnet{ + { + ID: "test-id", + Zone: "test-zone", + ZoneID: "test-zone-id", + }, + } + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1beta1ec2nodeclass.Status.Subnets { + Expect(v1ec2nodeclass.Status.Subnets[i].ID).To(Equal(v1beta1ec2nodeclass.Status.Subnets[i].ID)) + Expect(v1ec2nodeclass.Status.Subnets[i].Zone).To(Equal(v1beta1ec2nodeclass.Status.Subnets[i].Zone)) + Expect(v1ec2nodeclass.Status.Subnets[i].ZoneID).To(Equal(v1beta1ec2nodeclass.Status.Subnets[i].ZoneID)) + } + }) + It("should convert v1beta1 ec2nodeclass security group ", func() { + v1beta1ec2nodeclass.Status.SecurityGroups = []v1beta1.SecurityGroup{ + { + ID: "test-id", + Name: "test-name", + }, + } + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1beta1ec2nodeclass.Status.SecurityGroups { + Expect(v1ec2nodeclass.Status.SecurityGroups[i].ID).To(Equal(v1beta1ec2nodeclass.Status.SecurityGroups[i].ID)) + Expect(v1ec2nodeclass.Status.SecurityGroups[i].Name).To(Equal(v1beta1ec2nodeclass.Status.SecurityGroups[i].Name)) + } + }) + It("should convert v1beta1 ec2nodeclass ami", func() { + v1beta1ec2nodeclass.Status.AMIs = []v1beta1.AMI{ + { + ID: "test-id", + Name: "test-name", + }, + } + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + for i := range v1beta1ec2nodeclass.Status.AMIs { + Expect(v1ec2nodeclass.Status.AMIs[i].ID).To(Equal(v1beta1ec2nodeclass.Status.AMIs[i].ID)) + Expect(v1ec2nodeclass.Status.AMIs[i].Name).To(Equal(v1beta1ec2nodeclass.Status.AMIs[i].Name)) + + } + }) + It("should convert v1beta1 ec2nodeclass instance profile", func() { + v1beta1ec2nodeclass.Status.InstanceProfile = "test-instance-profile" + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1ec2nodeclass.Status.InstanceProfile).To(Equal(v1beta1ec2nodeclass.Status.InstanceProfile)) + }) + It("should convert v1beta1 ec2nodeclass conditions", func() { + v1beta1ec2nodeclass.Status.Conditions = []status.Condition{ + { + Status: status.ConditionReady, + Reason: "test-reason", + }, + } + Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed()) + Expect(v1ec2nodeclass.Status.Conditions).To(Equal(v1beta1ec2nodeclass.Status.Conditions)) + }) + }) +}) diff --git a/pkg/apis/v1/ec2nodeclass_defaults.go b/pkg/apis/v1/ec2nodeclass_defaults.go new file mode 100644 index 000000000000..8820e836bd0e --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass_defaults.go @@ -0,0 +1,22 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" +) + +// SetDefaults for the EC2NodeClass +func (in *EC2NodeClass) SetDefaults(_ context.Context) {} diff --git a/pkg/apis/v1/ec2nodeclass_hash_test.go b/pkg/apis/v1/ec2nodeclass_hash_test.go new file mode 100644 index 000000000000..baf0a0c7f115 --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass_hash_test.go @@ -0,0 +1,210 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1_test + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/imdario/mergo" + "github.com/samber/lo" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/karpenter/pkg/test" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Hash", func() { + const staticHash = "4950366118253097694" + var nodeClass *v1.EC2NodeClass + BeforeEach(func() { + nodeClass = &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + Role: "role-1", + Tags: map[string]string{ + "keyTag-1": "valueTag-1", + "keyTag-2": "valueTag-2", + }, + Context: lo.ToPtr("fake-context"), + DetailedMonitoring: lo.ToPtr(false), + AssociatePublicIPAddress: lo.ToPtr(false), + MetadataOptions: &v1.MetadataOptions{ + HTTPEndpoint: lo.ToPtr("disabled"), + HTTPProtocolIPv6: lo.ToPtr("disabled"), + HTTPPutResponseHopLimit: lo.ToPtr(int64(1)), + HTTPTokens: lo.ToPtr("optional"), + }, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: lo.ToPtr("map-device-1"), + RootVolume: false, + EBS: &v1.BlockDevice{ + DeleteOnTermination: lo.ToPtr(false), + Encrypted: lo.ToPtr(false), + IOPS: lo.ToPtr(int64(0)), + KMSKeyID: lo.ToPtr("fakeKMSKeyID"), + SnapshotID: lo.ToPtr("fakeSnapshot"), + Throughput: lo.ToPtr(int64(0)), + VolumeSize: resource.NewScaledQuantity(2, resource.Giga), + VolumeType: lo.ToPtr("standard"), + }, + }, + { + DeviceName: lo.ToPtr("map-device-2"), + }, + }, + UserData: aws.String("userdata-test-1"), + }, + } + }) + DescribeTable( + "should match static hash on field value change", + func(hash string, changes v1.EC2NodeClass) { + Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride, mergo.WithSliceDeepCopy)).To(Succeed()) + Expect(nodeClass.Hash()).To(Equal(hash)) + }, + Entry("Base EC2NodeClass", staticHash, v1.EC2NodeClass{}), + // Static fields, expect changed hash from base + + Entry("UserData", "9034828637236670345", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), + Entry("Tags", "6878220270322275255", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + Entry("Context", "13953931752662869657", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Context: aws.String("context-2")}}), + Entry("DetailedMonitoring", "14187487647319890991", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), + Entry("InstanceStorePolicy", "4160809219257698490", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{InstanceStorePolicy: lo.ToPtr(v1.InstanceStorePolicyRAID0)}}), + Entry("AssociatePublicIPAddress", "4469320567057431454", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{AssociatePublicIPAddress: lo.ToPtr(true)}}), + Entry("MetadataOptions HTTPEndpoint", "1277386558528601282", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPEndpoint: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPProtocolIPv6", "14697047633165484196", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPProtocolIPv6: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPPutResponseHopLimit", "2086799014304536137", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPPutResponseHopLimit: lo.ToPtr(int64(10))}}}), + Entry("MetadataOptions HTTPTokens", "14750841460622248593", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPTokens: lo.ToPtr("required")}}}), + Entry("BlockDeviceMapping DeviceName", "11716516558705174498", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{DeviceName: lo.ToPtr("map-device-test-3")}}}}), + Entry("BlockDeviceMapping RootVolume", "11900810786014401721", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{RootVolume: true}}}}), + Entry("BlockDeviceMapping DeleteOnTermination", "14586255897156659742", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{DeleteOnTermination: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping Encrypted", "10872029821841773628", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{Encrypted: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping IOPS", "9202874311950700210", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{IOPS: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping KMSKeyID", "14601456769467439478", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{KMSKeyID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping SnapshotID", "8031059801598053215", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{SnapshotID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping Throughput", "14410045481146650034", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{Throughput: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping VolumeType", "9480251663542054235", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{VolumeType: lo.ToPtr("io1")}}}}}), + + // Behavior / Dynamic fields, expect same hash as base + Entry("Modified AMISelector", staticHash, v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{AMISelectorTerms: []v1.AMISelectorTerm{{Tags: map[string]string{"": "ami-test-value"}}}}}), + Entry("Modified SubnetSelector", staticHash, v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{SubnetSelectorTerms: []v1.SubnetSelectorTerm{{Tags: map[string]string{"subnet-test-key": "subnet-test-value"}}}}}), + Entry("Modified SecurityGroupSelector", staticHash, v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{{Tags: map[string]string{"security-group-test-key": "security-group-test-value"}}}}}), + ) + // We create a separate test for updating blockDeviceMapping volumeSize, since resource.Quantity is a struct, and mergo.WithSliceDeepCopy + // doesn't work well with unexported fields, like the ones that are present in resource.Quantity + It("should match static hash when updating blockDeviceMapping volumeSize", func() { + nodeClass.Spec.BlockDeviceMappings[0].EBS.VolumeSize = resource.NewScaledQuantity(10, resource.Giga) + Expect(nodeClass.Hash()).To(Equal("5906178522470964189")) + }) + It("should match static hash for instanceProfile", func() { + nodeClass.Spec.Role = "" + nodeClass.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + Expect(nodeClass.Hash()).To(Equal("5855570904022890593")) + }) + It("should match static hash when reordering tags", func() { + nodeClass.Spec.Tags = map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"} + Expect(nodeClass.Hash()).To(Equal(staticHash)) + }) + It("should match static hash when reordering blockDeviceMappings", func() { + nodeClass.Spec.BlockDeviceMappings[0], nodeClass.Spec.BlockDeviceMappings[1] = nodeClass.Spec.BlockDeviceMappings[1], nodeClass.Spec.BlockDeviceMappings[0] + Expect(nodeClass.Hash()).To(Equal(staticHash)) + }) + DescribeTable("should change hash when static fields are updated", func(changes v1.EC2NodeClass) { + hash := nodeClass.Hash() + Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride, mergo.WithSliceDeepCopy)).To(Succeed()) + updatedHash := nodeClass.Hash() + Expect(hash).ToNot(Equal(updatedHash)) + }, + Entry("UserData", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), + Entry("Tags", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + Entry("Context", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Context: aws.String("context-2")}}), + Entry("DetailedMonitoring", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), + Entry("InstanceStorePolicy", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{InstanceStorePolicy: lo.ToPtr(v1.InstanceStorePolicyRAID0)}}), + Entry("AssociatePublicIPAddress", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{AssociatePublicIPAddress: lo.ToPtr(true)}}), + Entry("MetadataOptions HTTPEndpoint", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPEndpoint: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPProtocolIPv6", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPProtocolIPv6: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPPutResponseHopLimit", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPPutResponseHopLimit: lo.ToPtr(int64(10))}}}), + Entry("MetadataOptions HTTPTokens", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPTokens: lo.ToPtr("required")}}}), + Entry("BlockDeviceMapping DeviceName", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{DeviceName: lo.ToPtr("map-device-test-3")}}}}), + Entry("BlockDeviceMapping RootVolume", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{RootVolume: true}}}}), + Entry("BlockDeviceMapping DeleteOnTermination", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{DeleteOnTermination: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping Encrypted", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{Encrypted: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping IOPS", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{IOPS: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping KMSKeyID", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{KMSKeyID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping SnapshotID", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{SnapshotID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping Throughput", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{Throughput: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping VolumeType", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{VolumeType: lo.ToPtr("io1")}}}}}), + ) + // We create a separate test for updating blockDeviceMapping volumeSize, since resource.Quantity is a struct, and mergo.WithSliceDeepCopy + // doesn't work well with unexported fields, like the ones that are present in resource.Quantity + It("should change hash blockDeviceMapping volumeSize is updated", func() { + hash := nodeClass.Hash() + nodeClass.Spec.BlockDeviceMappings[0].EBS.VolumeSize = resource.NewScaledQuantity(10, resource.Giga) + updatedHash := nodeClass.Hash() + Expect(hash).ToNot(Equal(updatedHash)) + }) + It("should change hash when instanceProfile is updated", func() { + nodeClass.Spec.Role = "" + nodeClass.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + hash := nodeClass.Hash() + nodeClass.Spec.InstanceProfile = lo.ToPtr("other-instance-profile") + updatedHash := nodeClass.Hash() + Expect(hash).ToNot(Equal(updatedHash)) + }) + It("should not change hash when tags are re-ordered", func() { + hash := nodeClass.Hash() + nodeClass.Spec.Tags = map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"} + updatedHash := nodeClass.Hash() + Expect(hash).To(Equal(updatedHash)) + }) + It("should not change hash when blockDeviceMappings are re-ordered", func() { + hash := nodeClass.Hash() + nodeClass.Spec.BlockDeviceMappings[0], nodeClass.Spec.BlockDeviceMappings[1] = nodeClass.Spec.BlockDeviceMappings[1], nodeClass.Spec.BlockDeviceMappings[0] + updatedHash := nodeClass.Hash() + Expect(hash).To(Equal(updatedHash)) + }) + It("should not change hash when behavior/dynamic fields are updated", func() { + hash := nodeClass.Hash() + + // Update a behavior/dynamic field + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{"subnet-test-key": "subnet-test-value"}, + }, + } + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"sg-test-key": "sg-test-value"}, + }, + } + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{"ami-test-key": "ami-test-value"}, + }, + } + updatedHash := nodeClass.Hash() + Expect(hash).To(Equal(updatedHash)) + }) + It("should expect two EC2NodeClasses with the same spec to have the same hash", func() { + otherNodeClass := &v1.EC2NodeClass{ + Spec: nodeClass.Spec, + } + Expect(nodeClass.Hash()).To(Equal(otherNodeClass.Hash())) + }) +}) diff --git a/pkg/apis/v1/ec2nodeclass_status.go b/pkg/apis/v1/ec2nodeclass_status.go new file mode 100644 index 000000000000..89afe2370e8c --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass_status.go @@ -0,0 +1,102 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "github.com/awslabs/operatorpkg/status" + corev1 "k8s.io/api/core/v1" +) + +const ( + ConditionTypeSubnetsReady = "SubnetsReady" + ConditionTypeSecurityGroupsReady = "SecurityGroupsReady" + ConditionTypeAMIsReady = "AMIsReady" + ConditionTypeInstanceProfileReady = "InstanceProfileReady" +) + +// Subnet contains resolved Subnet selector values utilized for node launch +type Subnet struct { + // ID of the subnet + // +required + ID string `json:"id"` + // The associated availability zone + // +required + Zone string `json:"zone"` + // The associated availability zone ID + // +optional + ZoneID string `json:"zoneID,omitempty"` +} + +// SecurityGroup contains resolved SecurityGroup selector values utilized for node launch +type SecurityGroup struct { + // ID of the security group + // +required + ID string `json:"id"` + // Name of the security group + // +optional + Name string `json:"name,omitempty"` +} + +// AMI contains resolved AMI selector values utilized for node launch +type AMI struct { + // ID of the AMI + // +required + ID string `json:"id"` + // Name of the AMI + // +optional + Name string `json:"name,omitempty"` + // Requirements of the AMI to be utilized on an instance type + // +required + Requirements []corev1.NodeSelectorRequirement `json:"requirements"` +} + +// EC2NodeClassStatus contains the resolved state of the EC2NodeClass +type EC2NodeClassStatus struct { + // Subnets contains the current Subnet values that are available to the + // cluster under the subnet selectors. + // +optional + Subnets []Subnet `json:"subnets,omitempty"` + // SecurityGroups contains the current Security Groups values that are available to the + // cluster under the SecurityGroups selectors. + // +optional + SecurityGroups []SecurityGroup `json:"securityGroups,omitempty"` + // AMI contains the current AMI values that are available to the + // cluster under the AMI selectors. + // +optional + AMIs []AMI `json:"amis,omitempty"` + // InstanceProfile contains the resolved instance profile for the role + // +optional + InstanceProfile string `json:"instanceProfile,omitempty"` + // Conditions contains signals for health and readiness + // +optional + Conditions []status.Condition `json:"conditions,omitempty"` +} + +func (in *EC2NodeClass) StatusConditions() status.ConditionSet { + return status.NewReadyConditions( + ConditionTypeAMIsReady, + ConditionTypeSubnetsReady, + ConditionTypeSecurityGroupsReady, + ConditionTypeInstanceProfileReady, + ).For(in) +} + +func (in *EC2NodeClass) GetConditions() []status.Condition { + return in.Status.Conditions +} + +func (in *EC2NodeClass) SetConditions(conditions []status.Condition) { + in.Status.Conditions = conditions +} diff --git a/pkg/apis/v1/ec2nodeclass_validation_cel_test.go b/pkg/apis/v1/ec2nodeclass_validation_cel_test.go new file mode 100644 index 000000000000..1b926463fce1 --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass_validation_cel_test.go @@ -0,0 +1,1063 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1_test + +import ( + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/imdario/mergo" + "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "knative.dev/pkg/ptr" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + "sigs.k8s.io/karpenter/pkg/test" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("CEL/Validation", func() { + var nc *v1.EC2NodeClass + + BeforeEach(func() { + if env.Version.Minor() < 25 { + Skip("CEL Validation is for 1.25>") + } + nc = &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMIFamily: &v1.AMIFamilyAL2023, + AMISelectorTerms: []v1.AMISelectorTerm{{Alias: "al2023@latest"}}, + Role: "role-1", + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "*": "*", + }, + }, + }, + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "*": "*", + }, + }, + }, + }, + } + }) + It("should succeed if just specifying role", func() { + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed if just specifying instance profile", func() { + nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + nc.Spec.Role = "" + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail if specifying both instance profile and role", func() { + nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail if not specifying one of instance profile and role", func() { + nc.Spec.Role = "" + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + Context("UserData", func() { + It("should succeed if user data is empty", func() { + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + }) + Context("AMIFamily", func() { + amiFamilies := []string{v1.AMIFamilyAL2, v1.AMIFamilyAL2023, v1.AMIFamilyBottlerocket, v1.AMIFamilyWindows2019, v1.AMIFamilyWindows2022, v1.AMIFamilyCustom} + DescribeTable("should succeed with valid families", func() []interface{} { + f := func(amiFamily string) { + // Set a custom AMI family so it's compatible with all ami family types + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: "ami-0123456789abcdef"}} + nc.Spec.AMIFamily = lo.ToPtr(amiFamily) + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + } + entries := lo.Map(amiFamilies, func(family string, _ int) interface{} { + return Entry(family, family) + }) + return append([]interface{}{f}, entries...) + }()...) + It("should fail with the ubuntu family", func() { + // Set a custom AMI family so it's compatible with all ami family types + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: "ami-0123456789abcdef"}} + nc.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyUbuntu) + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + DescribeTable("should succeed when the amiFamily matches amiSelectorTerms[].alias", func() []interface{} { + f := func(amiFamily, alias string) { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: alias}} + nc.Spec.AMIFamily = lo.ToPtr(amiFamily) + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + } + entries := lo.FilterMap(amiFamilies, func(family string, _ int) (interface{}, bool) { + if family == v1.AMIFamilyCustom { + return nil, false + } + alias := fmt.Sprintf("%s@latest", strings.ToLower(family)) + return Entry( + fmt.Sprintf("family %q with alias %q", family, alias), + family, + alias, + ), true + }) + return append([]interface{}{f}, entries...) + }()...) + DescribeTable("should succeed when the amiFamily is custom with amiSelectorTerms[].alias", func() []interface{} { + f := func(alias string) { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: alias}} + nc.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + } + entries := lo.FilterMap(amiFamilies, func(family string, _ int) (interface{}, bool) { + if family == v1.AMIFamilyCustom { + return nil, false + } + alias := fmt.Sprintf("%s@latest", strings.ToLower(family)) + return Entry( + fmt.Sprintf(`family "Custom" with alias %q`, alias), + alias, + ), true + }) + return append([]interface{}{f}, entries...) + }()...) + DescribeTable("should fail when then amiFamily does not match amiSelectorTerms[].alias", func() []interface{} { + f := func(amiFamily, alias string) { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: alias}} + nc.Spec.AMIFamily = lo.ToPtr(amiFamily) + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + } + entries := []interface{}{} + families := lo.Reject(amiFamilies, func(family string, _ int) bool { + return family == v1.AMIFamilyCustom + }) + for i := range families { + for j := range families { + if i == j { + continue + } + alias := fmt.Sprintf("%s@latest", strings.ToLower(families[j])) + entries = append(entries, Entry( + fmt.Sprintf("family %q with alias %q", families[i], alias), + families[i], + alias, + )) + } + + } + return append([]interface{}{f}, entries...) + }()...) + It("should fail when neither amiFamily nor an alias are specified", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: "ami-01234567890abcdef"}} + nc.Spec.AMIFamily = nil + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("Tags", func() { + It("should succeed when tags are empty", func() { + nc.Spec.Tags = map[string]string{} + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed if tags aren't in restricted tag keys", func() { + nc.Spec.Tags = map[string]string{ + "karpenter.sh/custom-key": "value", + "karpenter.sh/managed": "true", + "kubernetes.io/role/key": "value", + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail if tags contain a restricted domain key", func() { + nc.Spec.Tags = map[string]string{ + karpv1.NodePoolLabelKey: "value", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + "kubernetes.io/cluster/test": "value", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + v1.EKSClusterNameTagKey: "test", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + v1.LabelNodeClass: "test", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + "karpenter.sh/nodeclaim": "test", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + }) + }) + Context("SubnetSelectorTerms", func() { + It("should succeed with a valid subnet selector on tags", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid subnet selector on id", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + ID: "subnet-12345749", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail when subnet selector terms is set to nil", func() { + nc.Spec.SubnetSelectorTerms = nil + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when no subnet selector terms exist", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{} + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has no values", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + {}, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has no tag map values", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has a tag map key that is empty", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has a tag map value that is empty", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when the last subnet selector is invalid", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + ID: "subnet-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("SecurityGroupSelectorTerms", func() { + It("should succeed with a valid security group selector on tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid security group selector on id", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid security group selector on name", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Name: "testname", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail when security group selector terms is set to nil", func() { + nc.Spec.SecurityGroupSelectorTerms = nil + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when no security group selector terms exist", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{} + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has no values", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + {}, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has no tag map values", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has a tag map key that is empty", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has a tag map value that is empty", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when the last security group selector is invalid", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with name", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + Name: "my-security-group", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying name with tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Name: "my-security-group", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("AMISelectorTerms", func() { + It("should succeed with a valid ami selector on alias", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ + Alias: "al2023@latest", + }} + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid ami selector on tags", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid ami selector on id", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + ID: "ami-12345749", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid ami selector on name", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Name: "testname", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid ami selector on name and owner", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Name: "testname", + Owner: "testowner", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed when an ami selector term has an owner key with tags", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Owner: "testowner", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail when a ami selector term has no values", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + {}, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has no tag map values", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has a tag map key that is empty", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has a tag map value that is empty", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when the last ami selector is invalid", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + DescribeTable( + "should fail when specifying id with other fields", + func(mutation v1.AMISelectorTerm) { + term := v1.AMISelectorTerm{ID: "ami-1234749"} + Expect(mergo.Merge(&term, &mutation)).To(Succeed()) + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{term} + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }, + Entry("alias", v1.AMISelectorTerm{Alias: "al2023@latest"}), + Entry("tags", v1.AMISelectorTerm{ + Tags: map[string]string{"test": "testvalue"}, + }), + Entry("name", v1.AMISelectorTerm{Name: "my-custom-ami"}), + Entry("owner", v1.AMISelectorTerm{Owner: "123456789"}), + ) + DescribeTable( + "should fail when specifying alias with other fields", + func(mutation v1.AMISelectorTerm) { + term := v1.AMISelectorTerm{Alias: "al2023@latest"} + Expect(mergo.Merge(&term, &mutation)).To(Succeed()) + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{term} + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }, + Entry("id", v1.AMISelectorTerm{ID: "ami-1234749"}), + Entry("tags", v1.AMISelectorTerm{ + Tags: map[string]string{"test": "testvalue"}, + }), + Entry("name", v1.AMISelectorTerm{Name: "my-custom-ami"}), + Entry("owner", v1.AMISelectorTerm{Owner: "123456789"}), + ) + It("should fail when specifying alias with other terms", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + {Alias: "al2023@latest"}, + {ID: "ami-1234749"}, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + DescribeTable( + "should succeed for valid aliases", + func(alias, family string) { + nc.Spec.AMIFamily = lo.ToPtr(family) + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: alias}} + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }, + Entry("al2 (latest)", "al2@latest", v1.AMIFamilyAL2), + Entry("al2 (pinned)", "al2@v20240625", v1.AMIFamilyAL2), + Entry("al2023 (latest)", "al2023@latest", v1.AMIFamilyAL2023), + Entry("al2023 (pinned)", "al2023@v20240625", v1.AMIFamilyAL2023), + Entry("bottlerocket (latest)", "bottlerocket@latest", v1.AMIFamilyBottlerocket), + Entry("bottlerocket (pinned)", "bottlerocket@1.10.0", v1.AMIFamilyBottlerocket), + Entry("windows2019 (latest)", "windows2019@latest", v1.AMIFamilyWindows2019), + Entry("windows2022 (latest)", "windows2022@latest", v1.AMIFamilyWindows2022), + ) + DescribeTable( + "should fail for incorrectly formatted aliases", + func(aliases ...string) { + for _, alias := range aliases { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: alias}} + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + } + }, + Entry("missing family", "@latest"), + Entry("missing version", "al2023", "al2023@"), + Entry("invalid separator", "al2023-latest"), + ) + It("should fail for an alias with an invalid family", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "ubuntu@latest"}} + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + DescribeTable( + "should fail when specifying non-latest versions with Windows aliases", + func(alias string) { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: alias}} + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }, + Entry("Windows2019", "windows2019@v1.0.0"), + Entry("Windows2022", "windows2022@v1.0.0"), + ) + }) + Context("Kubelet", func() { + It("should fail on kubeReserved with invalid keys", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + KubeReserved: map[string]string{ + string(corev1.ResourcePods): "2", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail on systemReserved with invalid keys", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + SystemReserved: map[string]string{ + string(corev1.ResourcePods): "2", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + Context("Eviction Signals", func() { + Context("Eviction Hard", func() { + It("should succeed on evictionHard with valid keys", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionHard: map[string]string{ + "memory.available": "5%", + "nodefs.available": "10%", + "nodefs.inodesFree": "15%", + "imagefs.available": "5%", + "imagefs.inodesFree": "5%", + "pid.available": "5%", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail on evictionHard with invalid keys", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionHard: map[string]string{ + "memory": "5%", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail on invalid formatted percentage value in evictionHard", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionHard: map[string]string{ + "memory.available": "5%3", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail on invalid percentage value (too large) in evictionHard", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionHard: map[string]string{ + "memory.available": "110%", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail on invalid quantity value in evictionHard", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionHard: map[string]string{ + "memory.available": "110GB", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + }) + Context("Eviction Soft", func() { + It("should succeed on evictionSoft with valid keys", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionSoft: map[string]string{ + "memory.available": "5%", + "nodefs.available": "10%", + "nodefs.inodesFree": "15%", + "imagefs.available": "5%", + "imagefs.inodesFree": "5%", + "pid.available": "5%", + }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "memory.available": {Duration: time.Minute}, + "nodefs.available": {Duration: time.Second * 90}, + "nodefs.inodesFree": {Duration: time.Minute * 5}, + "imagefs.available": {Duration: time.Hour}, + "imagefs.inodesFree": {Duration: time.Hour * 24}, + "pid.available": {Duration: time.Minute}, + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail on evictionSoft with invalid keys", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionSoft: map[string]string{ + "memory": "5%", + }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "memory": {Duration: time.Minute}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail on invalid formatted percentage value in evictionSoft", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionSoft: map[string]string{ + "memory.available": "5%3", + }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "memory.available": {Duration: time.Minute}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail on invalid percentage value (too large) in evictionSoft", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionSoft: map[string]string{ + "memory.available": "110%", + }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "memory.available": {Duration: time.Minute}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail on invalid quantity value in evictionSoft", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionSoft: map[string]string{ + "memory.available": "110GB", + }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "memory.available": {Duration: time.Minute}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when eviction soft doesn't have matching grace period", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionSoft: map[string]string{ + "memory.available": "200Mi", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("GCThresholdPercent", func() { + It("should succeed on a valid imageGCHighThresholdPercent", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + ImageGCHighThresholdPercent: ptr.Int32(10), + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail when imageGCHighThresholdPercent is less than imageGCLowThresholdPercent", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + ImageGCHighThresholdPercent: ptr.Int32(50), + ImageGCLowThresholdPercent: ptr.Int32(60), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when imageGCLowThresholdPercent is greather than imageGCHighThresheldPercent", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + ImageGCHighThresholdPercent: ptr.Int32(50), + ImageGCLowThresholdPercent: ptr.Int32(60), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("Eviction Soft Grace Period", func() { + It("should succeed on evictionSoftGracePeriod with valid keys", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionSoft: map[string]string{ + "memory.available": "5%", + "nodefs.available": "10%", + "nodefs.inodesFree": "15%", + "imagefs.available": "5%", + "imagefs.inodesFree": "5%", + "pid.available": "5%", + }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "memory.available": {Duration: time.Minute}, + "nodefs.available": {Duration: time.Second * 90}, + "nodefs.inodesFree": {Duration: time.Minute * 5}, + "imagefs.available": {Duration: time.Hour}, + "imagefs.inodesFree": {Duration: time.Hour * 24}, + "pid.available": {Duration: time.Minute}, + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail on evictionSoftGracePeriod with invalid keys", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "memory": {Duration: time.Minute}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when eviction soft grace period doesn't have matching threshold", func() { + nc.Spec.Kubelet = &v1.KubeletConfiguration{ + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "memory.available": {Duration: time.Minute}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + }) + Context("MetadataOptions", func() { + It("should succeed for valid inputs", func() { + nc.Spec.MetadataOptions = &v1.MetadataOptions{ + HTTPEndpoint: aws.String("disabled"), + HTTPProtocolIPv6: aws.String("enabled"), + HTTPPutResponseHopLimit: aws.Int64(34), + HTTPTokens: aws.String("optional"), + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail for invalid for HTTPEndpoint", func() { + nc.Spec.MetadataOptions = &v1.MetadataOptions{ + HTTPEndpoint: aws.String("test"), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail for invalid for HTTPProtocolIPv6", func() { + nc.Spec.MetadataOptions = &v1.MetadataOptions{ + HTTPProtocolIPv6: aws.String("test"), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail for invalid for HTTPPutResponseHopLimit", func() { + nc.Spec.MetadataOptions = &v1.MetadataOptions{ + HTTPPutResponseHopLimit: aws.Int64(-5), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail for invalid for HTTPTokens", func() { + nc.Spec.MetadataOptions = &v1.MetadataOptions{ + HTTPTokens: aws.String("test"), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("BlockDeviceMappings", func() { + It("should succeed if more than one root volume is specified", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: nc.Spec.AMISelectorTerms, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(500, resource.Giga), + }, + + RootVolume: true, + }, + { + DeviceName: aws.String("map-device-2"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(50, resource.Tera), + }, + + RootVolume: false, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) + }) + It("should succeed for valid VolumeSize in G", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: nc.Spec.AMISelectorTerms, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(58, resource.Giga), + }, + RootVolume: false, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) + }) + It("should succeed for valid VolumeSize in T", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: nc.Spec.AMISelectorTerms, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(45, resource.Tera), + }, + RootVolume: false, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) + }) + It("should fail if more than one root volume is specified", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: nc.Spec.AMISelectorTerms, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(50, resource.Giga), + }, + RootVolume: true, + }, + { + DeviceName: aws.String("map-device-2"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(50, resource.Giga), + }, + RootVolume: true, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) + }) + It("should fail VolumeSize is less then 1Gi/1G", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: nc.Spec.AMISelectorTerms, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(1, resource.Milli), + }, + RootVolume: false, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) + }) + It("should fail VolumeSize is greater then 64T", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: nc.Spec.AMISelectorTerms, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(100, resource.Tera), + }, + RootVolume: false, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) + }) + It("should fail for VolumeSize that do not parse into quantity values", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: nc.Spec.AMISelectorTerms, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: &resource.Quantity{}, + }, + RootVolume: false, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) + }) + }) + Context("Role Immutability", func() { + It("should fail if role is not defined", func() { + nc.Spec.Role = "" + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when updating the role", func() { + nc.Spec.Role = "test-role" + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + + nc.Spec.Role = "test-role2" + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail to switch between an unmanaged and managed instance profile", func() { + nc.Spec.Role = "" + nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + + nc.Spec.Role = "test-role" + nc.Spec.InstanceProfile = nil + Expect(env.Client.Update(ctx, nc)).ToNot(Succeed()) + }) + It("should fail to switch between a managed and unmanaged instance profile", func() { + nc.Spec.Role = "test-role" + nc.Spec.InstanceProfile = nil + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + + nc.Spec.Role = "" + nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + Expect(env.Client.Update(ctx, nc)).ToNot(Succeed()) + }) + }) +}) diff --git a/pkg/apis/v1/labels.go b/pkg/apis/v1/labels.go new file mode 100644 index 000000000000..c391b0faab01 --- /dev/null +++ b/pkg/apis/v1/labels.go @@ -0,0 +1,137 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "fmt" + "regexp" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + coreapis "sigs.k8s.io/karpenter/pkg/apis" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + + "github.com/aws/karpenter-provider-aws/pkg/apis" +) + +func init() { + karpv1.RestrictedLabelDomains = karpv1.RestrictedLabelDomains.Insert(RestrictedLabelDomains...) + karpv1.WellKnownLabels = karpv1.WellKnownLabels.Insert( + LabelInstanceHypervisor, + LabelInstanceEncryptionInTransitSupported, + LabelInstanceCategory, + LabelInstanceFamily, + LabelInstanceGeneration, + LabelInstanceSize, + LabelInstanceLocalNVME, + LabelInstanceCPU, + LabelInstanceCPUManufacturer, + LabelInstanceMemory, + LabelInstanceEBSBandwidth, + LabelInstanceNetworkBandwidth, + LabelInstanceGPUName, + LabelInstanceGPUManufacturer, + LabelInstanceGPUCount, + LabelInstanceGPUMemory, + LabelInstanceAcceleratorName, + LabelInstanceAcceleratorManufacturer, + LabelInstanceAcceleratorCount, + LabelTopologyZoneID, + corev1.LabelWindowsBuild, + ) +} + +var ( + TerminationFinalizer = apis.Group + "/termination" + AWSToKubeArchitectures = map[string]string{ + "x86_64": karpv1.ArchitectureAmd64, + karpv1.ArchitectureArm64: karpv1.ArchitectureArm64, + } + WellKnownArchitectures = sets.NewString( + karpv1.ArchitectureAmd64, + karpv1.ArchitectureArm64, + ) + RestrictedLabelDomains = []string{ + apis.Group, + } + RestrictedTagPatterns = []*regexp.Regexp{ + // Adheres to cluster name pattern matching as specified in the API spec + // https://docs.aws.amazon.com/eks/latest/APIReference/API_CreateCluster.html + regexp.MustCompile(`^kubernetes\.io/cluster/[0-9A-Za-z][A-Za-z0-9\-_]*$`), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(karpv1.NodePoolLabelKey))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(EKSClusterNameTagKey))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(LabelNodeClass))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(TagNodeClaim))), + } + AMIFamilyBottlerocket = "Bottlerocket" + AMIFamilyAL2 = "AL2" + AMIFamilyAL2023 = "AL2023" + AMIFamilyUbuntu = "Ubuntu" + AMIFamilyWindows2019 = "Windows2019" + AMIFamilyWindows2022 = "Windows2022" + AMIFamilyCustom = "Custom" + Windows2019 = "2019" + Windows2022 = "2022" + WindowsCore = "Core" + Windows2019Build = "10.0.17763" + Windows2022Build = "10.0.20348" + ResourceNVIDIAGPU corev1.ResourceName = "nvidia.com/gpu" + ResourceAMDGPU corev1.ResourceName = "amd.com/gpu" + ResourceAWSNeuron corev1.ResourceName = "aws.amazon.com/neuron" + ResourceHabanaGaudi corev1.ResourceName = "habana.ai/gaudi" + ResourceAWSPodENI corev1.ResourceName = "vpc.amazonaws.com/pod-eni" + ResourcePrivateIPv4Address corev1.ResourceName = "vpc.amazonaws.com/PrivateIPv4Address" + ResourceEFA corev1.ResourceName = "vpc.amazonaws.com/efa" + + EKSClusterNameTagKey = "eks:eks-cluster-name" + + LabelNodeClass = apis.Group + "/ec2nodeclass" + + LabelTopologyZoneID = "topology.k8s.aws/zone-id" + + LabelInstanceHypervisor = apis.Group + "/instance-hypervisor" + LabelInstanceEncryptionInTransitSupported = apis.Group + "/instance-encryption-in-transit-supported" + LabelInstanceCategory = apis.Group + "/instance-category" + LabelInstanceFamily = apis.Group + "/instance-family" + LabelInstanceGeneration = apis.Group + "/instance-generation" + LabelInstanceLocalNVME = apis.Group + "/instance-local-nvme" + LabelInstanceSize = apis.Group + "/instance-size" + LabelInstanceCPU = apis.Group + "/instance-cpu" + LabelInstanceCPUManufacturer = apis.Group + "/instance-cpu-manufacturer" + LabelInstanceMemory = apis.Group + "/instance-memory" + LabelInstanceEBSBandwidth = apis.Group + "/instance-ebs-bandwidth" + LabelInstanceNetworkBandwidth = apis.Group + "/instance-network-bandwidth" + LabelInstanceGPUName = apis.Group + "/instance-gpu-name" + LabelInstanceGPUManufacturer = apis.Group + "/instance-gpu-manufacturer" + LabelInstanceGPUCount = apis.Group + "/instance-gpu-count" + LabelInstanceGPUMemory = apis.Group + "/instance-gpu-memory" + LabelInstanceAcceleratorName = apis.Group + "/instance-accelerator-name" + LabelInstanceAcceleratorManufacturer = apis.Group + "/instance-accelerator-manufacturer" + LabelInstanceAcceleratorCount = apis.Group + "/instance-accelerator-count" + AnnotationEC2NodeClassHash = apis.Group + "/ec2nodeclass-hash" + AnnotationKubeletCompatibilityHash = apis.CompatibilityGroup + "/kubelet-drift-hash" + AnnotationClusterNameTaggedCompatability = apis.CompatibilityGroup + "/cluster-name-tagged" + AnnotationEC2NodeClassHashVersion = apis.Group + "/ec2nodeclass-hash-version" + AnnotationInstanceTagged = apis.Group + "/tagged" + + AnnotationUbuntuCompatibilityKey = apis.CompatibilityGroup + "/v1beta1-ubuntu" + AnnotationUbuntuCompatibilityIncompatible = "incompatible" + AnnotationUbuntuCompatibilityAMIFamily = "amiFamily" + AnnotationUbuntuCompatibilityBlockDeviceMappings = "blockDeviceMappings" + + TagNodeClaim = coreapis.Group + "/nodeclaim" + TagManagedLaunchTemplate = apis.Group + "/cluster" + TagName = "Name" +) diff --git a/pkg/apis/v1/nodepool_validation_cel_test.go b/pkg/apis/v1/nodepool_validation_cel_test.go new file mode 100644 index 000000000000..bd1c84cf79e1 --- /dev/null +++ b/pkg/apis/v1/nodepool_validation_cel_test.go @@ -0,0 +1,112 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1_test + +import ( + "strings" + + "github.com/Pallinder/go-randomdata" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" +) + +var _ = Describe("CEL/Validation", func() { + var nodePool *karpv1.NodePool + + BeforeEach(func() { + if env.Version.Minor() < 25 { + Skip("CEL Validation is for 1.25>") + } + nodePool = &karpv1.NodePool{ + ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: "karpenter.k8s.aws", + Kind: "EC2NodeClass", + Name: "default", + }, + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpExists, + }, + }, + }, + }, + }, + }, + } + }) + Context("Requirements", func() { + It("should allow restricted domains exceptions", func() { + oldNodePool := nodePool.DeepCopy() + for label := range karpv1.LabelDomainExceptions { + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: label + "/test", Operator: corev1.NodeSelectorOpIn, Values: []string{"test"}}}, + } + Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) + Expect(nodePool.RuntimeValidate()).To(Succeed()) + Expect(env.Client.Delete(ctx, nodePool)).To(Succeed()) + nodePool = oldNodePool.DeepCopy() + } + }) + It("should allow well known label exceptions", func() { + oldNodePool := nodePool.DeepCopy() + for label := range karpv1.WellKnownLabels.Difference(sets.New(karpv1.NodePoolLabelKey)) { + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: label, Operator: corev1.NodeSelectorOpIn, Values: []string{"test"}}}, + } + Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) + Expect(nodePool.RuntimeValidate()).To(Succeed()) + Expect(env.Client.Delete(ctx, nodePool)).To(Succeed()) + nodePool = oldNodePool.DeepCopy() + } + }) + }) + Context("Labels", func() { + It("should allow restricted domains exceptions", func() { + oldNodePool := nodePool.DeepCopy() + for label := range karpv1.LabelDomainExceptions { + nodePool.Spec.Template.Labels = map[string]string{ + label: "test", + } + Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) + Expect(nodePool.RuntimeValidate()).To(Succeed()) + Expect(env.Client.Delete(ctx, nodePool)).To(Succeed()) + nodePool = oldNodePool.DeepCopy() + } + }) + It("should allow well known label exceptions", func() { + oldNodePool := nodePool.DeepCopy() + for label := range karpv1.WellKnownLabels.Difference(sets.New(karpv1.NodePoolLabelKey)) { + nodePool.Spec.Template.Labels = map[string]string{ + label: "test", + } + Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) + Expect(nodePool.RuntimeValidate()).To(Succeed()) + Expect(env.Client.Delete(ctx, nodePool)).To(Succeed()) + nodePool = oldNodePool.DeepCopy() + } + }) + }) +}) diff --git a/pkg/apis/v1/suite_test.go b/pkg/apis/v1/suite_test.go new file mode 100644 index 000000000000..79375cd7cf85 --- /dev/null +++ b/pkg/apis/v1/suite_test.go @@ -0,0 +1,56 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1_test + +import ( + "context" + "testing" + + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "knative.dev/pkg/logging/testing" + + . "sigs.k8s.io/karpenter/pkg/test/expectations" + + coretest "sigs.k8s.io/karpenter/pkg/test" + + "github.com/aws/karpenter-provider-aws/pkg/apis" + "github.com/aws/karpenter-provider-aws/pkg/test" +) + +var ctx context.Context +var env *coretest.Environment +var awsEnv *test.Environment + +func TestAPIs(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Validation") +} + +var _ = BeforeSuite(func() { + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) + awsEnv = test.NewEnvironment(ctx, env) +}) + +var _ = AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) +}) + +var _ = AfterSuite(func() { + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) diff --git a/pkg/apis/v1/zz_generated.deepcopy.go b/pkg/apis/v1/zz_generated.deepcopy.go new file mode 100644 index 000000000000..627c14cfad13 --- /dev/null +++ b/pkg/apis/v1/zz_generated.deepcopy.go @@ -0,0 +1,541 @@ +//go:build !ignore_autogenerated + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1 + +import ( + "github.com/awslabs/operatorpkg/status" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AMI) DeepCopyInto(out *AMI) { + *out = *in + if in.Requirements != nil { + in, out := &in.Requirements, &out.Requirements + *out = make([]corev1.NodeSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AMI. +func (in *AMI) DeepCopy() *AMI { + if in == nil { + return nil + } + out := new(AMI) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AMISelectorTerm) DeepCopyInto(out *AMISelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AMISelectorTerm. +func (in *AMISelectorTerm) DeepCopy() *AMISelectorTerm { + if in == nil { + return nil + } + out := new(AMISelectorTerm) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlockDevice) DeepCopyInto(out *BlockDevice) { + *out = *in + if in.DeleteOnTermination != nil { + in, out := &in.DeleteOnTermination, &out.DeleteOnTermination + *out = new(bool) + **out = **in + } + if in.Encrypted != nil { + in, out := &in.Encrypted, &out.Encrypted + *out = new(bool) + **out = **in + } + if in.IOPS != nil { + in, out := &in.IOPS, &out.IOPS + *out = new(int64) + **out = **in + } + if in.KMSKeyID != nil { + in, out := &in.KMSKeyID, &out.KMSKeyID + *out = new(string) + **out = **in + } + if in.SnapshotID != nil { + in, out := &in.SnapshotID, &out.SnapshotID + *out = new(string) + **out = **in + } + if in.Throughput != nil { + in, out := &in.Throughput, &out.Throughput + *out = new(int64) + **out = **in + } + if in.VolumeSize != nil { + in, out := &in.VolumeSize, &out.VolumeSize + x := (*in).DeepCopy() + *out = &x + } + if in.VolumeType != nil { + in, out := &in.VolumeType, &out.VolumeType + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockDevice. +func (in *BlockDevice) DeepCopy() *BlockDevice { + if in == nil { + return nil + } + out := new(BlockDevice) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlockDeviceMapping) DeepCopyInto(out *BlockDeviceMapping) { + *out = *in + if in.DeviceName != nil { + in, out := &in.DeviceName, &out.DeviceName + *out = new(string) + **out = **in + } + if in.EBS != nil { + in, out := &in.EBS, &out.EBS + *out = new(BlockDevice) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockDeviceMapping. +func (in *BlockDeviceMapping) DeepCopy() *BlockDeviceMapping { + if in == nil { + return nil + } + out := new(BlockDeviceMapping) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EC2NodeClass) DeepCopyInto(out *EC2NodeClass) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EC2NodeClass. +func (in *EC2NodeClass) DeepCopy() *EC2NodeClass { + if in == nil { + return nil + } + out := new(EC2NodeClass) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EC2NodeClass) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EC2NodeClassList) DeepCopyInto(out *EC2NodeClassList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EC2NodeClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EC2NodeClassList. +func (in *EC2NodeClassList) DeepCopy() *EC2NodeClassList { + if in == nil { + return nil + } + out := new(EC2NodeClassList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EC2NodeClassList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EC2NodeClassSpec) DeepCopyInto(out *EC2NodeClassSpec) { + *out = *in + if in.SubnetSelectorTerms != nil { + in, out := &in.SubnetSelectorTerms, &out.SubnetSelectorTerms + *out = make([]SubnetSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SecurityGroupSelectorTerms != nil { + in, out := &in.SecurityGroupSelectorTerms, &out.SecurityGroupSelectorTerms + *out = make([]SecurityGroupSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.AssociatePublicIPAddress != nil { + in, out := &in.AssociatePublicIPAddress, &out.AssociatePublicIPAddress + *out = new(bool) + **out = **in + } + if in.AMISelectorTerms != nil { + in, out := &in.AMISelectorTerms, &out.AMISelectorTerms + *out = make([]AMISelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.AMIFamily != nil { + in, out := &in.AMIFamily, &out.AMIFamily + *out = new(string) + **out = **in + } + if in.UserData != nil { + in, out := &in.UserData, &out.UserData + *out = new(string) + **out = **in + } + if in.InstanceProfile != nil { + in, out := &in.InstanceProfile, &out.InstanceProfile + *out = new(string) + **out = **in + } + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Kubelet != nil { + in, out := &in.Kubelet, &out.Kubelet + *out = new(KubeletConfiguration) + (*in).DeepCopyInto(*out) + } + if in.BlockDeviceMappings != nil { + in, out := &in.BlockDeviceMappings, &out.BlockDeviceMappings + *out = make([]*BlockDeviceMapping, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(BlockDeviceMapping) + (*in).DeepCopyInto(*out) + } + } + } + if in.InstanceStorePolicy != nil { + in, out := &in.InstanceStorePolicy, &out.InstanceStorePolicy + *out = new(InstanceStorePolicy) + **out = **in + } + if in.DetailedMonitoring != nil { + in, out := &in.DetailedMonitoring, &out.DetailedMonitoring + *out = new(bool) + **out = **in + } + if in.MetadataOptions != nil { + in, out := &in.MetadataOptions, &out.MetadataOptions + *out = new(MetadataOptions) + (*in).DeepCopyInto(*out) + } + if in.Context != nil { + in, out := &in.Context, &out.Context + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EC2NodeClassSpec. +func (in *EC2NodeClassSpec) DeepCopy() *EC2NodeClassSpec { + if in == nil { + return nil + } + out := new(EC2NodeClassSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EC2NodeClassStatus) DeepCopyInto(out *EC2NodeClassStatus) { + *out = *in + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make([]Subnet, len(*in)) + copy(*out, *in) + } + if in.SecurityGroups != nil { + in, out := &in.SecurityGroups, &out.SecurityGroups + *out = make([]SecurityGroup, len(*in)) + copy(*out, *in) + } + if in.AMIs != nil { + in, out := &in.AMIs, &out.AMIs + *out = make([]AMI, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]status.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EC2NodeClassStatus. +func (in *EC2NodeClassStatus) DeepCopy() *EC2NodeClassStatus { + if in == nil { + return nil + } + out := new(EC2NodeClassStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { + *out = *in + if in.ClusterDNS != nil { + in, out := &in.ClusterDNS, &out.ClusterDNS + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MaxPods != nil { + in, out := &in.MaxPods, &out.MaxPods + *out = new(int32) + **out = **in + } + if in.PodsPerCore != nil { + in, out := &in.PodsPerCore, &out.PodsPerCore + *out = new(int32) + **out = **in + } + if in.SystemReserved != nil { + in, out := &in.SystemReserved, &out.SystemReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.KubeReserved != nil { + in, out := &in.KubeReserved, &out.KubeReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionHard != nil { + in, out := &in.EvictionHard, &out.EvictionHard + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionSoft != nil { + in, out := &in.EvictionSoft, &out.EvictionSoft + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionSoftGracePeriod != nil { + in, out := &in.EvictionSoftGracePeriod, &out.EvictionSoftGracePeriod + *out = make(map[string]metav1.Duration, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionMaxPodGracePeriod != nil { + in, out := &in.EvictionMaxPodGracePeriod, &out.EvictionMaxPodGracePeriod + *out = new(int32) + **out = **in + } + if in.ImageGCHighThresholdPercent != nil { + in, out := &in.ImageGCHighThresholdPercent, &out.ImageGCHighThresholdPercent + *out = new(int32) + **out = **in + } + if in.ImageGCLowThresholdPercent != nil { + in, out := &in.ImageGCLowThresholdPercent, &out.ImageGCLowThresholdPercent + *out = new(int32) + **out = **in + } + if in.CPUCFSQuota != nil { + in, out := &in.CPUCFSQuota, &out.CPUCFSQuota + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletConfiguration. +func (in *KubeletConfiguration) DeepCopy() *KubeletConfiguration { + if in == nil { + return nil + } + out := new(KubeletConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetadataOptions) DeepCopyInto(out *MetadataOptions) { + *out = *in + if in.HTTPEndpoint != nil { + in, out := &in.HTTPEndpoint, &out.HTTPEndpoint + *out = new(string) + **out = **in + } + if in.HTTPProtocolIPv6 != nil { + in, out := &in.HTTPProtocolIPv6, &out.HTTPProtocolIPv6 + *out = new(string) + **out = **in + } + if in.HTTPPutResponseHopLimit != nil { + in, out := &in.HTTPPutResponseHopLimit, &out.HTTPPutResponseHopLimit + *out = new(int64) + **out = **in + } + if in.HTTPTokens != nil { + in, out := &in.HTTPTokens, &out.HTTPTokens + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetadataOptions. +func (in *MetadataOptions) DeepCopy() *MetadataOptions { + if in == nil { + return nil + } + out := new(MetadataOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityGroup) DeepCopyInto(out *SecurityGroup) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityGroup. +func (in *SecurityGroup) DeepCopy() *SecurityGroup { + if in == nil { + return nil + } + out := new(SecurityGroup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityGroupSelectorTerm) DeepCopyInto(out *SecurityGroupSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityGroupSelectorTerm. +func (in *SecurityGroupSelectorTerm) DeepCopy() *SecurityGroupSelectorTerm { + if in == nil { + return nil + } + out := new(SecurityGroupSelectorTerm) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Subnet) DeepCopyInto(out *Subnet) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Subnet. +func (in *Subnet) DeepCopy() *Subnet { + if in == nil { + return nil + } + out := new(Subnet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetSelectorTerm) DeepCopyInto(out *SubnetSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetSelectorTerm. +func (in *SubnetSelectorTerm) DeepCopy() *SubnetSelectorTerm { + if in == nil { + return nil + } + out := new(SubnetSelectorTerm) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/apis/v1beta1/doc.go b/pkg/apis/v1beta1/doc.go index 93f6c319052f..bfbe13887f53 100644 --- a/pkg/apis/v1beta1/doc.go +++ b/pkg/apis/v1beta1/doc.go @@ -17,3 +17,20 @@ limitations under the License. // +k8s:defaulter-gen=TypeMeta // +groupName=karpenter.k8s.aws package v1beta1 // doc.go is discovered by codegen + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" + + "github.com/aws/karpenter-provider-aws/pkg/apis" +) + +func init() { + gv := schema.GroupVersion{Group: apis.Group, Version: "v1beta1"} + metav1.AddToGroupVersion(scheme.Scheme, gv) + scheme.Scheme.AddKnownTypes(gv, + &EC2NodeClass{}, + &EC2NodeClassList{}, + ) +} diff --git a/pkg/apis/v1beta1/ec2nodeclass.go b/pkg/apis/v1beta1/ec2nodeclass.go index f9515b5e4d2f..833dcdd5683e 100644 --- a/pkg/apis/v1beta1/ec2nodeclass.go +++ b/pkg/apis/v1beta1/ec2nodeclass.go @@ -21,7 +21,7 @@ import ( "github.com/samber/lo" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" ) // EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. @@ -108,9 +108,9 @@ type EC2NodeClassSpec struct { // (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) // for limiting exposure of Instance Metadata and User Data to pods. // If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 - // disabled, with httpPutResponseLimit of 2, and with httpTokens + // disabled, with httpPutResponseLimit of 1, and with httpTokens // required. - // +kubebuilder:default={"httpEndpoint":"enabled","httpProtocolIPv6":"disabled","httpPutResponseHopLimit":2,"httpTokens":"required"} + // +kubebuilder:default={"httpEndpoint":"enabled","httpProtocolIPv6":"disabled","httpPutResponseHopLimit":1,"httpTokens":"required"} // +optional MetadataOptions *MetadataOptions `json:"metadataOptions,omitempty"` // Context is a Reserved field in EC2 APIs @@ -292,7 +292,8 @@ type BlockDevice struct { // + TODO: Add the CEL resources.quantity type after k8s 1.29 // + https://github.com/kubernetes/apiserver/commit/b137c256373aec1c5d5810afbabb8932a19ecd2a#diff-838176caa5882465c9d6061febd456397a3e2b40fb423ed36f0cabb1847ecb4dR190 // +kubebuilder:validation:Pattern:="^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$" - // +kubebuilder:validation:XIntOrString + // +kubebuilder:validation:Schemaless + // +kubebuilder:validation:Type:=string // +optional VolumeSize *resource.Quantity `json:"volumeSize,omitempty" hash:"string"` // VolumeType of the block device. @@ -355,7 +356,7 @@ func (in *EC2NodeClass) InstanceProfileRole() string { func (in *EC2NodeClass) InstanceProfileTags(clusterName string) map[string]string { return lo.Assign(in.Spec.Tags, map[string]string{ fmt.Sprintf("kubernetes.io/cluster/%s", clusterName): "owned", - corev1beta1.ManagedByAnnotationKey: clusterName, + karpv1beta1.ManagedByAnnotationKey: clusterName, LabelNodeClass: in.Name, }) } diff --git a/pkg/apis/v1beta1/ec2nodeclass_conversion.go b/pkg/apis/v1beta1/ec2nodeclass_conversion.go new file mode 100644 index 000000000000..b10d91cace27 --- /dev/null +++ b/pkg/apis/v1beta1/ec2nodeclass_conversion.go @@ -0,0 +1,27 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + + "knative.dev/pkg/apis" +) + +// Since v1 is the hub conversion version, We will only need to implement conversion webhooks for v1 + +func (in *EC2NodeClass) ConvertTo(ctx context.Context, to apis.Convertible) error { return nil } + +func (in *EC2NodeClass) ConvertFrom(ctx context.Context, from apis.Convertible) error { return nil } diff --git a/pkg/apis/v1beta1/ec2nodeclass_hash_test.go b/pkg/apis/v1beta1/ec2nodeclass_hash_test.go index ccdde5a389ec..311e34ac564d 100644 --- a/pkg/apis/v1beta1/ec2nodeclass_hash_test.go +++ b/pkg/apis/v1beta1/ec2nodeclass_hash_test.go @@ -31,7 +31,7 @@ var _ = Describe("Hash", func() { const staticHash = "10790156025840984195" var nodeClass *v1beta1.EC2NodeClass BeforeEach(func() { - nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ + nodeClass = test.BetaEC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ AMIFamily: lo.ToPtr(v1beta1.AMIFamilyAL2023), Role: "role-1", @@ -202,7 +202,7 @@ var _ = Describe("Hash", func() { Expect(hash).To(Equal(updatedHash)) }) It("should expect two EC2NodeClasses with the same spec to have the same hash", func() { - otherNodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + otherNodeClass := test.BetaEC2NodeClass(v1beta1.EC2NodeClass{ Spec: nodeClass.Spec, }) Expect(nodeClass.Hash()).To(Equal(otherNodeClass.Hash())) diff --git a/pkg/apis/v1beta1/ec2nodeclass_status.go b/pkg/apis/v1beta1/ec2nodeclass_status.go index 611e94d62117..fa89119b9bb2 100644 --- a/pkg/apis/v1beta1/ec2nodeclass_status.go +++ b/pkg/apis/v1beta1/ec2nodeclass_status.go @@ -15,7 +15,15 @@ limitations under the License. package v1beta1 import ( - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "github.com/awslabs/operatorpkg/status" + corev1 "k8s.io/api/core/v1" +) + +const ( + ConditionTypeSubnetsReady = "SubnetsReady" + ConditionTypeSecurityGroupsReady = "SecurityGroupsReady" + ConditionTypeAMIsReady = "AMIsReady" + ConditionTypeInstanceProfileReady = "InstanceProfileReady" ) // Subnet contains resolved Subnet selector values utilized for node launch @@ -26,6 +34,9 @@ type Subnet struct { // The associated availability zone // +required Zone string `json:"zone"` + // The associated availability zone ID + // +optional + ZoneID string `json:"zoneID,omitempty"` } // SecurityGroup contains resolved SecurityGroup selector values utilized for node launch @@ -48,7 +59,7 @@ type AMI struct { Name string `json:"name,omitempty"` // Requirements of the AMI to be utilized on an instance type // +required - Requirements []corev1beta1.NodeSelectorRequirementWithMinValues `json:"requirements"` + Requirements []corev1.NodeSelectorRequirement `json:"requirements"` } // EC2NodeClassStatus contains the resolved state of the EC2NodeClass @@ -68,4 +79,24 @@ type EC2NodeClassStatus struct { // InstanceProfile contains the resolved instance profile for the role // +optional InstanceProfile string `json:"instanceProfile,omitempty"` + // Conditions contains signals for health and readiness + // +optional + Conditions []status.Condition `json:"conditions,omitempty"` +} + +func (in *EC2NodeClass) StatusConditions() status.ConditionSet { + return status.NewReadyConditions( + ConditionTypeAMIsReady, + ConditionTypeSubnetsReady, + ConditionTypeSecurityGroupsReady, + ConditionTypeInstanceProfileReady, + ).For(in) +} + +func (in *EC2NodeClass) GetConditions() []status.Condition { + return in.Status.Conditions +} + +func (in *EC2NodeClass) SetConditions(conditions []status.Condition) { + in.Status.Conditions = conditions } diff --git a/pkg/apis/v1beta1/ec2nodeclass_validation.go b/pkg/apis/v1beta1/ec2nodeclass_validation.go deleted file mode 100644 index 4fc18527f00d..000000000000 --- a/pkg/apis/v1beta1/ec2nodeclass_validation.go +++ /dev/null @@ -1,307 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "context" - "fmt" - "strings" - - "github.com/aws/aws-sdk-go/service/ec2" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" - "k8s.io/apimachinery/pkg/api/resource" - "knative.dev/pkg/apis" -) - -const ( - subnetSelectorTermsPath = "subnetSelectorTerms" - securityGroupSelectorTermsPath = "securityGroupSelectorTerms" - amiSelectorTermsPath = "amiSelectorTerms" - amiFamilyPath = "amiFamily" - tagsPath = "tags" - metadataOptionsPath = "metadataOptions" - blockDeviceMappingsPath = "blockDeviceMappings" - rolePath = "role" - instanceProfilePath = "instanceProfile" -) - -var ( - minVolumeSize = *resource.NewScaledQuantity(1, resource.Giga) - maxVolumeSize = *resource.NewScaledQuantity(64, resource.Tera) -) - -func (in *EC2NodeClass) SupportedVerbs() []admissionregistrationv1.OperationType { - return []admissionregistrationv1.OperationType{ - admissionregistrationv1.Create, - admissionregistrationv1.Update, - } -} - -func (in *EC2NodeClass) Validate(ctx context.Context) (errs *apis.FieldError) { - if apis.IsInUpdate(ctx) { - original := apis.GetBaseline(ctx).(*EC2NodeClass) - errs = in.validateImmutableFields(original) - } - return errs.Also( - apis.ValidateObjectMetadata(in).ViaField("metadata"), - in.Spec.validate(ctx).ViaField("spec"), - ) -} - -func (in *EC2NodeClass) validateImmutableFields(original *EC2NodeClass) (errs *apis.FieldError) { - return errs.Also( - in.Spec.validateRoleImmutability(&original.Spec).ViaField("spec"), - ) -} - -func (in *EC2NodeClassSpec) validate(_ context.Context) (errs *apis.FieldError) { - if in.Role != "" && in.InstanceProfile != nil { - errs = errs.Also(apis.ErrMultipleOneOf(rolePath, instanceProfilePath)) - } - if in.Role == "" && in.InstanceProfile == nil { - errs = errs.Also(apis.ErrMissingOneOf(rolePath, instanceProfilePath)) - } - return errs.Also( - in.validateSubnetSelectorTerms().ViaField(subnetSelectorTermsPath), - in.validateSecurityGroupSelectorTerms().ViaField(securityGroupSelectorTermsPath), - in.validateAMISelectorTerms().ViaField(amiSelectorTermsPath), - in.validateMetadataOptions().ViaField(metadataOptionsPath), - in.validateAMIFamily().ViaField(amiFamilyPath), - in.validateBlockDeviceMappings().ViaField(blockDeviceMappingsPath), - in.validateTags().ViaField(tagsPath), - ) -} - -func (in *EC2NodeClassSpec) validateSubnetSelectorTerms() (errs *apis.FieldError) { - if len(in.SubnetSelectorTerms) == 0 { - errs = errs.Also(apis.ErrMissingOneOf()) - } - for i, term := range in.SubnetSelectorTerms { - errs = errs.Also(term.validate()).ViaIndex(i) - } - return errs -} - -func (in *SubnetSelectorTerm) validate() (errs *apis.FieldError) { - errs = errs.Also(validateTags(in.Tags).ViaField("tags")) - if len(in.Tags) == 0 && in.ID == "" { - errs = errs.Also(apis.ErrGeneric("expected at least one, got none", "tags", "id")) - } else if in.ID != "" && len(in.Tags) > 0 { - errs = errs.Also(apis.ErrGeneric(`"id" is mutually exclusive, cannot be set with a combination of other fields in`)) - } - return errs -} - -func (in *EC2NodeClassSpec) validateSecurityGroupSelectorTerms() (errs *apis.FieldError) { - if len(in.SecurityGroupSelectorTerms) == 0 { - errs = errs.Also(apis.ErrMissingOneOf()) - } - for _, term := range in.SecurityGroupSelectorTerms { - errs = errs.Also(term.validate()) - } - return errs -} - -//nolint:gocyclo -func (in *SecurityGroupSelectorTerm) validate() (errs *apis.FieldError) { - errs = errs.Also(validateTags(in.Tags).ViaField("tags")) - if len(in.Tags) == 0 && in.ID == "" && in.Name == "" { - errs = errs.Also(apis.ErrGeneric("expect at least one, got none", "tags", "id", "name")) - } else if in.ID != "" && (len(in.Tags) > 0 || in.Name != "") { - errs = errs.Also(apis.ErrGeneric(`"id" is mutually exclusive, cannot be set with a combination of other fields in`)) - } else if in.Name != "" && (len(in.Tags) > 0 || in.ID != "") { - errs = errs.Also(apis.ErrGeneric(`"name" is mutually exclusive, cannot be set with a combination of other fields in`)) - } - return errs -} - -func (in *EC2NodeClassSpec) validateAMISelectorTerms() (errs *apis.FieldError) { - for _, term := range in.AMISelectorTerms { - errs = errs.Also(term.validate()) - } - return errs -} - -//nolint:gocyclo -func (in *AMISelectorTerm) validate() (errs *apis.FieldError) { - errs = errs.Also(validateTags(in.Tags).ViaField("tags")) - if len(in.Tags) == 0 && in.ID == "" && in.Name == "" { - errs = errs.Also(apis.ErrGeneric("expect at least one, got none", "tags", "id", "name")) - } else if in.ID != "" && (len(in.Tags) > 0 || in.Name != "" || in.Owner != "") { - errs = errs.Also(apis.ErrGeneric(`"id" is mutually exclusive, cannot be set with a combination of other fields in`)) - } - return errs -} - -func validateTags(m map[string]string) (errs *apis.FieldError) { - for k, v := range m { - if k == "" { - errs = errs.Also(apis.ErrInvalidKeyName(`""`, "")) - } - if v == "" { - errs = errs.Also(apis.ErrInvalidValue(`""`, k)) - } - } - return errs -} - -func (in *EC2NodeClassSpec) validateMetadataOptions() (errs *apis.FieldError) { - if in.MetadataOptions == nil { - return nil - } - return errs.Also( - in.validateHTTPEndpoint(), - in.validateHTTPProtocolIpv6(), - in.validateHTTPPutResponseHopLimit(), - in.validateHTTPTokens(), - ) -} - -func (in *EC2NodeClassSpec) validateHTTPEndpoint() *apis.FieldError { - if in.MetadataOptions.HTTPEndpoint == nil { - return nil - } - return in.validateStringEnum(*in.MetadataOptions.HTTPEndpoint, "httpEndpoint", ec2.LaunchTemplateInstanceMetadataEndpointState_Values()) -} - -func (in *EC2NodeClassSpec) validateHTTPProtocolIpv6() *apis.FieldError { - if in.MetadataOptions.HTTPProtocolIPv6 == nil { - return nil - } - return in.validateStringEnum(*in.MetadataOptions.HTTPProtocolIPv6, "httpProtocolIPv6", ec2.LaunchTemplateInstanceMetadataProtocolIpv6_Values()) -} - -func (in *EC2NodeClassSpec) validateHTTPPutResponseHopLimit() *apis.FieldError { - if in.MetadataOptions.HTTPPutResponseHopLimit == nil { - return nil - } - limit := *in.MetadataOptions.HTTPPutResponseHopLimit - if limit < 1 || limit > 64 { - return apis.ErrOutOfBoundsValue(limit, 1, 64, "httpPutResponseHopLimit") - } - return nil -} - -func (in *EC2NodeClassSpec) validateHTTPTokens() *apis.FieldError { - if in.MetadataOptions.HTTPTokens == nil { - return nil - } - return in.validateStringEnum(*in.MetadataOptions.HTTPTokens, "httpTokens", ec2.LaunchTemplateHttpTokensState_Values()) -} - -func (in *EC2NodeClassSpec) validateStringEnum(value, field string, validValues []string) *apis.FieldError { - for _, validValue := range validValues { - if value == validValue { - return nil - } - } - return apis.ErrInvalidValue(fmt.Sprintf("%s not in %v", value, strings.Join(validValues, ", ")), field) -} - -func (in *EC2NodeClassSpec) validateBlockDeviceMappings() (errs *apis.FieldError) { - numRootVolume := 0 - for i, blockDeviceMapping := range in.BlockDeviceMappings { - if err := in.validateBlockDeviceMapping(blockDeviceMapping); err != nil { - errs = errs.Also(err.ViaFieldIndex(blockDeviceMappingsPath, i)) - } - if blockDeviceMapping.RootVolume { - numRootVolume++ - } - } - if numRootVolume > 1 { - errs = errs.Also(apis.ErrMultipleOneOf("more than 1 root volume configured")) - } - return errs -} - -func (in *EC2NodeClassSpec) validateBlockDeviceMapping(blockDeviceMapping *BlockDeviceMapping) (errs *apis.FieldError) { - return errs.Also(in.validateDeviceName(blockDeviceMapping), in.validateEBS(blockDeviceMapping)) -} - -func (in *EC2NodeClassSpec) validateDeviceName(blockDeviceMapping *BlockDeviceMapping) *apis.FieldError { - if blockDeviceMapping.DeviceName == nil { - return apis.ErrMissingField("deviceName") - } - return nil -} - -func (in *EC2NodeClassSpec) validateEBS(blockDeviceMapping *BlockDeviceMapping) (errs *apis.FieldError) { - if blockDeviceMapping.EBS == nil { - return apis.ErrMissingField("ebs") - } - for _, err := range []*apis.FieldError{ - in.validateVolumeType(blockDeviceMapping), - in.validateVolumeSize(blockDeviceMapping), - } { - if err != nil { - errs = errs.Also(err.ViaField("ebs")) - } - } - return errs -} - -func (in *EC2NodeClassSpec) validateVolumeType(blockDeviceMapping *BlockDeviceMapping) *apis.FieldError { - if blockDeviceMapping.EBS.VolumeType != nil { - return in.validateStringEnum(*blockDeviceMapping.EBS.VolumeType, "volumeType", ec2.VolumeType_Values()) - } - return nil -} - -func (in *EC2NodeClassSpec) validateVolumeSize(blockDeviceMapping *BlockDeviceMapping) *apis.FieldError { - // If an EBS mapping is present, one of volumeSize or snapshotID must be present - if blockDeviceMapping.EBS.SnapshotID != nil && blockDeviceMapping.EBS.VolumeSize == nil { - return nil - } else if blockDeviceMapping.EBS.VolumeSize == nil { - return apis.ErrMissingField("volumeSize") - } else if blockDeviceMapping.EBS.VolumeSize.Cmp(minVolumeSize) == -1 || blockDeviceMapping.EBS.VolumeSize.Cmp(maxVolumeSize) == 1 { - return apis.ErrOutOfBoundsValue(blockDeviceMapping.EBS.VolumeSize.String(), minVolumeSize.String(), maxVolumeSize.String(), "volumeSize") - } - return nil -} - -func (in *EC2NodeClassSpec) validateAMIFamily() (errs *apis.FieldError) { - if in.AMIFamily == nil { - return nil - } - if *in.AMIFamily == AMIFamilyCustom && len(in.AMISelectorTerms) == 0 { - errs = errs.Also(apis.ErrMissingField(amiSelectorTermsPath)) - } - return errs -} - -func (in *EC2NodeClassSpec) validateTags() (errs *apis.FieldError) { - for k, v := range in.Tags { - if k == "" { - errs = errs.Also(apis.ErrInvalidValue(fmt.Sprintf( - "the tag with key : '' and value : '%s' is invalid because empty tag keys aren't supported", v), "tags")) - } - for _, pattern := range RestrictedTagPatterns { - if pattern.MatchString(k) { - errs = errs.Also(apis.ErrInvalidKeyName(k, "tags", fmt.Sprintf("tag contains in restricted tag matching %q", pattern.String()))) - } - } - } - return errs -} - -func (in *EC2NodeClassSpec) validateRoleImmutability(originalSpec *EC2NodeClassSpec) *apis.FieldError { - if in.Role != originalSpec.Role { - return &apis.FieldError{ - Message: "Immutable field changed", - Paths: []string{"role"}, - } - } - return nil -} diff --git a/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go b/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go index b0104c9c506a..0ccde246fb30 100644 --- a/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go +++ b/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go @@ -18,7 +18,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/samber/lo" "k8s.io/apimachinery/pkg/api/resource" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/test" @@ -34,7 +34,7 @@ var _ = Describe("CEL/Validation", func() { if env.Version.Minor() < 25 { Skip("CEL Validation is for 1.25>") } - nc = test.EC2NodeClass() + nc = test.BetaEC2NodeClass() }) It("should succeed if just specifying role", func() { Expect(env.Client.Create(ctx, nc)).To(Succeed()) @@ -72,7 +72,7 @@ var _ = Describe("CEL/Validation", func() { }) It("should fail if tags contain a restricted domain key", func() { nc.Spec.Tags = map[string]string{ - corev1beta1.NodePoolLabelKey: "value", + karpv1beta1.NodePoolLabelKey: "value", } Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) nc.Spec.Tags = map[string]string{ @@ -80,7 +80,7 @@ var _ = Describe("CEL/Validation", func() { } Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) nc.Spec.Tags = map[string]string{ - corev1beta1.ManagedByAnnotationKey: "test", + karpv1beta1.ManagedByAnnotationKey: "test", } Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) nc.Spec.Tags = map[string]string{ @@ -494,7 +494,7 @@ var _ = Describe("CEL/Validation", func() { }) Context("BlockDeviceMappings", func() { It("should succeed if more than one root volume is specified", func() { - nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + nodeClass := test.BetaEC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ { @@ -519,7 +519,7 @@ var _ = Describe("CEL/Validation", func() { Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) }) It("should succeed for valid VolumeSize in G", func() { - nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + nodeClass := test.BetaEC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ { @@ -535,7 +535,7 @@ var _ = Describe("CEL/Validation", func() { Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) }) It("should succeed for valid VolumeSize in T", func() { - nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + nodeClass := test.BetaEC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ { @@ -551,7 +551,7 @@ var _ = Describe("CEL/Validation", func() { Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) }) It("should fail if more than one root volume is specified", func() { - nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + nodeClass := test.BetaEC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ { @@ -574,7 +574,7 @@ var _ = Describe("CEL/Validation", func() { Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) }) It("should fail VolumeSize is less then 1Gi/1G", func() { - nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + nodeClass := test.BetaEC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ { @@ -590,7 +590,7 @@ var _ = Describe("CEL/Validation", func() { Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) }) It("should fail VolumeSize is greater then 64T", func() { - nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + nodeClass := test.BetaEC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ { @@ -606,7 +606,7 @@ var _ = Describe("CEL/Validation", func() { Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) }) It("should fail for VolumeSize that do not parse into quantity values", func() { - nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + nodeClass := test.BetaEC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ { diff --git a/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go b/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go deleted file mode 100644 index 18035649e235..000000000000 --- a/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go +++ /dev/null @@ -1,525 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1_test - -import ( - "github.com/samber/lo" - "k8s.io/apimachinery/pkg/api/resource" - "knative.dev/pkg/apis" - - "github.com/aws/aws-sdk-go/aws" - - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" - "github.com/aws/karpenter-provider-aws/pkg/test" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Webhook/Validation", func() { - var nc *v1beta1.EC2NodeClass - - BeforeEach(func() { - nc = test.EC2NodeClass() - }) - It("should succeed if just specifying role", func() { - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed if just specifying instance profile", func() { - nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") - nc.Spec.Role = "" - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should fail if specifying both instance profile and role", func() { - nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail if not specifying one of instance profile and role", func() { - nc.Spec.Role = "" - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - Context("UserData", func() { - It("should succeed if user data is empty", func() { - Expect(nc.Validate(ctx)).To(Succeed()) - }) - }) - Context("Tags", func() { - It("should succeed when tags are empty", func() { - nc.Spec.Tags = map[string]string{} - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed if tags aren't in restricted tag keys", func() { - nc.Spec.Tags = map[string]string{ - "karpenter.sh/custom-key": "value", - "karpenter.sh/managed": "true", - "kubernetes.io/role/key": "value", - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed by validating that regex is properly escaped", func() { - nc.Spec.Tags = map[string]string{ - "karpenterzsh/nodepool": "value", - } - Expect(nc.Validate(ctx)).To(Succeed()) - nc.Spec.Tags = map[string]string{ - "kubernetesbio/cluster/test": "value", - } - Expect(nc.Validate(ctx)).To(Succeed()) - nc.Spec.Tags = map[string]string{ - "karpenterzsh/managed-by": "test", - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should fail if tags contain a restricted domain key", func() { - nc.Spec.Tags = map[string]string{ - "karpenter.sh/nodepool": "value", - } - Expect(nc.Validate(ctx)).To(Not(Succeed())) - nc.Spec.Tags = map[string]string{ - "kubernetes.io/cluster/test": "value", - } - Expect(nc.Validate(ctx)).To(Not(Succeed())) - nc.Spec.Tags = map[string]string{ - "karpenter.sh/managed-by": "test", - } - Expect(nc.Validate(ctx)).To(Not(Succeed())) - nc.Spec.Tags = map[string]string{ - v1beta1.LabelNodeClass: "test", - } - Expect(nc.Validate(ctx)).To(Not(Succeed())) - nc.Spec.Tags = map[string]string{ - "karpenter.sh/nodeclaim": "test", - } - Expect(nc.Validate(ctx)).To(Not(Succeed())) - }) - }) - Context("SubnetSelectorTerms", func() { - It("should succeed with a valid subnet selector on tags", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed with a valid subnet selector on id", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - ID: "subnet-12345749", - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should fail when subnet selector terms is set to nil", func() { - nc.Spec.SubnetSelectorTerms = nil - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when no subnet selector terms exist", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{} - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a subnet selector term has no values", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - {}, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a subnet selector term has no tag map values", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - Tags: map[string]string{}, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a subnet selector term has a tag map key that is empty", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - Tags: map[string]string{ - "test": "", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a subnet selector term has a tag map value that is empty", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - Tags: map[string]string{ - "": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when the last subnet selector is invalid", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - Tags: map[string]string{ - "test": "testvalue", - }, - }, - { - Tags: map[string]string{ - "test2": "testvalue2", - }, - }, - { - Tags: map[string]string{ - "test3": "testvalue3", - }, - }, - { - Tags: map[string]string{ - "": "testvalue4", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying id with tags", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - ID: "subnet-12345749", - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - }) - Context("SecurityGroupSelectorTerms", func() { - It("should succeed with a valid security group selector on tags", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed with a valid security group selector on id", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - ID: "sg-12345749", - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed with a valid security group selector on name", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Name: "testname", - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should fail when security group selector terms is set to nil", func() { - nc.Spec.SecurityGroupSelectorTerms = nil - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when no security group selector terms exist", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{} - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a security group selector term has no values", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - {}, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a security group selector term has no tag map values", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Tags: map[string]string{}, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a security group selector term has a tag map key that is empty", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Tags: map[string]string{ - "test": "", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a security group selector term has a tag map value that is empty", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Tags: map[string]string{ - "": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when the last security group selector is invalid", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Tags: map[string]string{ - "test": "testvalue", - }, - }, - { - Tags: map[string]string{ - "test2": "testvalue2", - }, - }, - { - Tags: map[string]string{ - "test3": "testvalue3", - }, - }, - { - Tags: map[string]string{ - "": "testvalue4", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying id with tags", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - ID: "sg-12345749", - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying id with name", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - ID: "sg-12345749", - Name: "my-security-group", - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying name with tags", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Name: "my-security-group", - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - }) - Context("AMISelectorTerms", func() { - It("should succeed with a valid ami selector on tags", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed with a valid ami selector on id", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: "sg-12345749", - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed with a valid ami selector on name", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Name: "testname", - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed with a valid ami selector on name and owner", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Name: "testname", - Owner: "testowner", - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed when an ami selector term has an owner key with tags", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Owner: "testowner", - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should fail when a ami selector term has no values", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - {}, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a ami selector term has no tag map values", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Tags: map[string]string{}, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a ami selector term has a tag map key that is empty", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Tags: map[string]string{ - "test": "", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a ami selector term has a tag map value that is empty", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Tags: map[string]string{ - "": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when the last ami selector is invalid", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Tags: map[string]string{ - "test": "testvalue", - }, - }, - { - Tags: map[string]string{ - "test2": "testvalue2", - }, - }, - { - Tags: map[string]string{ - "test3": "testvalue3", - }, - }, - { - Tags: map[string]string{ - "": "testvalue4", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying id with tags", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: "ami-12345749", - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying id with name", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: "ami-12345749", - Name: "my-custom-ami", - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying id with owner", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: "ami-12345749", - Owner: "123456789", - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - }) - Context("BlockDeviceMappings", func() { - It("should fail if more than one root volume is specified", func() { - nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ - BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ - { - DeviceName: aws.String("map-device-1"), - EBS: &v1beta1.BlockDevice{ - VolumeSize: resource.NewScaledQuantity(50, resource.Giga), - }, - - RootVolume: true, - }, - { - DeviceName: aws.String("map-device-2"), - EBS: &v1beta1.BlockDevice{ - VolumeSize: resource.NewScaledQuantity(50, resource.Giga), - }, - - RootVolume: true, - }, - }, - }, - }) - Expect(nodeClass.Validate(ctx)).To(Not(Succeed())) - }) - }) - Context("Role Immutability", func() { - It("should fail when updating the role", func() { - nc.Spec.Role = "test-role" - Expect(nc.Validate(ctx)).To(Succeed()) - - updateCtx := apis.WithinUpdate(ctx, nc.DeepCopy()) - nc.Spec.Role = "test-role2" - Expect(nc.Validate(updateCtx)).ToNot(Succeed()) - }) - It("should fail to switch between an unmanaged and managed instance profile", func() { - nc.Spec.Role = "" - nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") - Expect(nc.Validate(ctx)).To(Succeed()) - - updateCtx := apis.WithinUpdate(ctx, nc.DeepCopy()) - nc.Spec.Role = "test-role" - nc.Spec.InstanceProfile = nil - Expect(nc.Validate(updateCtx)).ToNot(Succeed()) - }) - It("should fail to switch between a managed and unmanaged instance profile", func() { - nc.Spec.Role = "test-role" - nc.Spec.InstanceProfile = nil - Expect(nc.Validate(ctx)).To(Succeed()) - - updateCtx := apis.WithinUpdate(ctx, nc.DeepCopy()) - nc.Spec.Role = "" - nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") - Expect(nc.Validate(updateCtx)).ToNot(Succeed()) - }) - }) -}) diff --git a/pkg/apis/v1beta1/labels.go b/pkg/apis/v1beta1/labels.go index cadf8dd902c6..d7ae21c84230 100644 --- a/pkg/apis/v1beta1/labels.go +++ b/pkg/apis/v1beta1/labels.go @@ -18,15 +18,17 @@ import ( "fmt" "regexp" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" + coreapis "sigs.k8s.io/karpenter/pkg/apis" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/pkg/apis" ) func init() { - v1beta1.RestrictedLabelDomains = v1beta1.RestrictedLabelDomains.Insert(RestrictedLabelDomains...) - v1beta1.WellKnownLabels = v1beta1.WellKnownLabels.Insert( + karpv1beta1.RestrictedLabelDomains = karpv1beta1.RestrictedLabelDomains.Insert(RestrictedLabelDomains...) + karpv1beta1.WellKnownLabels = karpv1beta1.WellKnownLabels.Insert( LabelInstanceHypervisor, LabelInstanceEncryptionInTransitSupported, LabelInstanceCategory, @@ -37,6 +39,7 @@ func init() { LabelInstanceCPU, LabelInstanceCPUManufacturer, LabelInstanceMemory, + LabelInstanceEBSBandwidth, LabelInstanceNetworkBandwidth, LabelInstanceGPUName, LabelInstanceGPUManufacturer, @@ -45,79 +48,81 @@ func init() { LabelInstanceAcceleratorName, LabelInstanceAcceleratorManufacturer, LabelInstanceAcceleratorCount, - v1.LabelWindowsBuild, + LabelTopologyZoneID, + corev1.LabelWindowsBuild, ) } -const ( - TerminationFinalizer = Group + "/termination" -) - var ( + TerminationFinalizer = apis.Group + "/termination" AWSToKubeArchitectures = map[string]string{ - "x86_64": v1beta1.ArchitectureAmd64, - v1beta1.ArchitectureArm64: v1beta1.ArchitectureArm64, + "x86_64": karpv1beta1.ArchitectureAmd64, + karpv1beta1.ArchitectureArm64: karpv1beta1.ArchitectureArm64, } WellKnownArchitectures = sets.NewString( - v1beta1.ArchitectureAmd64, - v1beta1.ArchitectureArm64, + karpv1beta1.ArchitectureAmd64, + karpv1beta1.ArchitectureArm64, ) RestrictedLabelDomains = []string{ - Group, + apis.Group, } RestrictedTagPatterns = []*regexp.Regexp{ // Adheres to cluster name pattern matching as specified in the API spec // https://docs.aws.amazon.com/eks/latest/APIReference/API_CreateCluster.html regexp.MustCompile(`^kubernetes\.io/cluster/[0-9A-Za-z][A-Za-z0-9\-_]*$`), - regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1beta1.NodePoolLabelKey))), - regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1beta1.ManagedByAnnotationKey))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(karpv1beta1.NodePoolLabelKey))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(karpv1beta1.ManagedByAnnotationKey))), regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(LabelNodeClass))), regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(TagNodeClaim))), } - AMIFamilyBottlerocket = "Bottlerocket" - AMIFamilyAL2 = "AL2" - AMIFamilyAL2023 = "AL2023" - AMIFamilyUbuntu = "Ubuntu" - AMIFamilyWindows2019 = "Windows2019" - AMIFamilyWindows2022 = "Windows2022" - AMIFamilyCustom = "Custom" - Windows2019 = "2019" - Windows2022 = "2022" - WindowsCore = "Core" - Windows2019Build = "10.0.17763" - Windows2022Build = "10.0.20348" - ResourceNVIDIAGPU v1.ResourceName = "nvidia.com/gpu" - ResourceAMDGPU v1.ResourceName = "amd.com/gpu" - ResourceAWSNeuron v1.ResourceName = "aws.amazon.com/neuron" - ResourceHabanaGaudi v1.ResourceName = "habana.ai/gaudi" - ResourceAWSPodENI v1.ResourceName = "vpc.amazonaws.com/pod-eni" - ResourcePrivateIPv4Address v1.ResourceName = "vpc.amazonaws.com/PrivateIPv4Address" - ResourceEFA v1.ResourceName = "vpc.amazonaws.com/efa" + AMIFamilyBottlerocket = "Bottlerocket" + AMIFamilyAL2 = "AL2" + AMIFamilyAL2023 = "AL2023" + AMIFamilyUbuntu = "Ubuntu" + AMIFamilyWindows2019 = "Windows2019" + AMIFamilyWindows2022 = "Windows2022" + AMIFamilyCustom = "Custom" + Windows2019 = "2019" + Windows2022 = "2022" + WindowsCore = "Core" + Windows2019Build = "10.0.17763" + Windows2022Build = "10.0.20348" + ResourceNVIDIAGPU corev1.ResourceName = "nvidia.com/gpu" + ResourceAMDGPU corev1.ResourceName = "amd.com/gpu" + ResourceAWSNeuron corev1.ResourceName = "aws.amazon.com/neuron" + ResourceHabanaGaudi corev1.ResourceName = "habana.ai/gaudi" + ResourceAWSPodENI corev1.ResourceName = "vpc.amazonaws.com/pod-eni" + ResourcePrivateIPv4Address corev1.ResourceName = "vpc.amazonaws.com/PrivateIPv4Address" + ResourceEFA corev1.ResourceName = "vpc.amazonaws.com/efa" + + LabelNodeClass = apis.Group + "/ec2nodeclass" - LabelNodeClass = Group + "/ec2nodeclass" + LabelTopologyZoneID = "topology.k8s.aws/zone-id" - LabelInstanceHypervisor = Group + "/instance-hypervisor" - LabelInstanceEncryptionInTransitSupported = Group + "/instance-encryption-in-transit-supported" - LabelInstanceCategory = Group + "/instance-category" - LabelInstanceFamily = Group + "/instance-family" - LabelInstanceGeneration = Group + "/instance-generation" - LabelInstanceLocalNVME = Group + "/instance-local-nvme" - LabelInstanceSize = Group + "/instance-size" - LabelInstanceCPU = Group + "/instance-cpu" - LabelInstanceCPUManufacturer = Group + "/instance-cpu-manufacturer" - LabelInstanceMemory = Group + "/instance-memory" - LabelInstanceNetworkBandwidth = Group + "/instance-network-bandwidth" - LabelInstanceGPUName = Group + "/instance-gpu-name" - LabelInstanceGPUManufacturer = Group + "/instance-gpu-manufacturer" - LabelInstanceGPUCount = Group + "/instance-gpu-count" - LabelInstanceGPUMemory = Group + "/instance-gpu-memory" - LabelInstanceAcceleratorName = Group + "/instance-accelerator-name" - LabelInstanceAcceleratorManufacturer = Group + "/instance-accelerator-manufacturer" - LabelInstanceAcceleratorCount = Group + "/instance-accelerator-count" - AnnotationEC2NodeClassHash = Group + "/ec2nodeclass-hash" - AnnotationEC2NodeClassHashVersion = Group + "/ec2nodeclass-hash-version" - AnnotationInstanceTagged = Group + "/tagged" + LabelInstanceHypervisor = apis.Group + "/instance-hypervisor" + LabelInstanceEncryptionInTransitSupported = apis.Group + "/instance-encryption-in-transit-supported" + LabelInstanceCategory = apis.Group + "/instance-category" + LabelInstanceFamily = apis.Group + "/instance-family" + LabelInstanceGeneration = apis.Group + "/instance-generation" + LabelInstanceLocalNVME = apis.Group + "/instance-local-nvme" + LabelInstanceSize = apis.Group + "/instance-size" + LabelInstanceCPU = apis.Group + "/instance-cpu" + LabelInstanceCPUManufacturer = apis.Group + "/instance-cpu-manufacturer" + LabelInstanceMemory = apis.Group + "/instance-memory" + LabelInstanceEBSBandwidth = apis.Group + "/instance-ebs-bandwidth" + LabelInstanceNetworkBandwidth = apis.Group + "/instance-network-bandwidth" + LabelInstanceGPUName = apis.Group + "/instance-gpu-name" + LabelInstanceGPUManufacturer = apis.Group + "/instance-gpu-manufacturer" + LabelInstanceGPUCount = apis.Group + "/instance-gpu-count" + LabelInstanceGPUMemory = apis.Group + "/instance-gpu-memory" + LabelInstanceAcceleratorName = apis.Group + "/instance-accelerator-name" + LabelInstanceAcceleratorManufacturer = apis.Group + "/instance-accelerator-manufacturer" + LabelInstanceAcceleratorCount = apis.Group + "/instance-accelerator-count" + AnnotationEC2NodeClassHash = apis.Group + "/ec2nodeclass-hash" + AnnotationEC2NodeClassHashVersion = apis.Group + "/ec2nodeclass-hash-version" + AnnotationInstanceTagged = apis.Group + "/tagged" - TagNodeClaim = v1beta1.Group + "/nodeclaim" - TagName = "Name" + TagNodeClaim = coreapis.Group + "/nodeclaim" + TagManagedLaunchTemplate = apis.Group + "/cluster" + TagName = "Name" ) diff --git a/pkg/apis/v1beta1/nodepool_validation_cel_test.go b/pkg/apis/v1beta1/nodepool_validation_cel_test.go index 574e8a0779f9..9ac288cb3454 100644 --- a/pkg/apis/v1beta1/nodepool_validation_cel_test.go +++ b/pkg/apis/v1beta1/nodepool_validation_cel_test.go @@ -20,34 +20,35 @@ import ( "github.com/Pallinder/go-randomdata" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" ) var _ = Describe("CEL/Validation", func() { - var nodePool *v1beta1.NodePool + var nodePool *karpv1beta1.NodePool BeforeEach(func() { if env.Version.Minor() < 25 { Skip("CEL Validation is for 1.25>") } - nodePool = &v1beta1.NodePool{ + nodePool = &karpv1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - NodeClassRef: &v1beta1.NodeClassReference{ - Kind: "NodeClaim", - Name: "default", + Spec: karpv1beta1.NodePoolSpec{ + Template: karpv1beta1.NodeClaimTemplate{ + Spec: karpv1beta1.NodeClaimSpec{ + NodeClassRef: &karpv1beta1.NodeClassReference{ + APIVersion: "karpenter.k8s.aws/v1beta1", + Kind: "EC2NodeClass", + Name: "default", }, - Requirements: []v1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []karpv1beta1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpExists, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1beta1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpExists, }, }, }, @@ -59,9 +60,9 @@ var _ = Describe("CEL/Validation", func() { Context("Requirements", func() { It("should allow restricted domains exceptions", func() { oldNodePool := nodePool.DeepCopy() - for label := range v1beta1.LabelDomainExceptions { - nodePool.Spec.Template.Spec.Requirements = []v1beta1.NodeSelectorRequirementWithMinValues{ - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}, + for label := range karpv1beta1.LabelDomainExceptions { + nodePool.Spec.Template.Spec.Requirements = []karpv1beta1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: label + "/test", Operator: corev1.NodeSelectorOpIn, Values: []string{"test"}}}, } Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) Expect(nodePool.RuntimeValidate()).To(Succeed()) @@ -71,9 +72,9 @@ var _ = Describe("CEL/Validation", func() { }) It("should allow well known label exceptions", func() { oldNodePool := nodePool.DeepCopy() - for label := range v1beta1.WellKnownLabels.Difference(sets.New(v1beta1.NodePoolLabelKey)) { - nodePool.Spec.Template.Spec.Requirements = []v1beta1.NodeSelectorRequirementWithMinValues{ - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}, + for label := range karpv1beta1.WellKnownLabels.Difference(sets.New(karpv1beta1.NodePoolLabelKey)) { + nodePool.Spec.Template.Spec.Requirements = []karpv1beta1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: label, Operator: corev1.NodeSelectorOpIn, Values: []string{"test"}}}, } Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) Expect(nodePool.RuntimeValidate()).To(Succeed()) @@ -85,7 +86,7 @@ var _ = Describe("CEL/Validation", func() { Context("Labels", func() { It("should allow restricted domains exceptions", func() { oldNodePool := nodePool.DeepCopy() - for label := range v1beta1.LabelDomainExceptions { + for label := range karpv1beta1.LabelDomainExceptions { nodePool.Spec.Template.Labels = map[string]string{ label: "test", } @@ -97,7 +98,7 @@ var _ = Describe("CEL/Validation", func() { }) It("should allow well known label exceptions", func() { oldNodePool := nodePool.DeepCopy() - for label := range v1beta1.WellKnownLabels.Difference(sets.New(v1beta1.NodePoolLabelKey)) { + for label := range karpv1beta1.WellKnownLabels.Difference(sets.New(karpv1beta1.NodePoolLabelKey)) { nodePool.Spec.Template.Labels = map[string]string{ label: "test", } diff --git a/pkg/apis/v1beta1/suite_test.go b/pkg/apis/v1beta1/suite_test.go index c610ab795189..5ec90893e7c1 100644 --- a/pkg/apis/v1beta1/suite_test.go +++ b/pkg/apis/v1beta1/suite_test.go @@ -18,13 +18,14 @@ import ( "context" "testing" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" @@ -42,7 +43,7 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) awsEnv = test.NewEnvironment(ctx, env) }) diff --git a/pkg/apis/v1beta1/zz_generated.deepcopy.go b/pkg/apis/v1beta1/zz_generated.deepcopy.go index 781d88c876c8..64c04b8c31f8 100644 --- a/pkg/apis/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/v1beta1/zz_generated.deepcopy.go @@ -19,8 +19,9 @@ limitations under the License. package v1beta1 import ( - "k8s.io/apimachinery/pkg/runtime" - apisv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "github.com/awslabs/operatorpkg/status" + "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -28,7 +29,7 @@ func (in *AMI) DeepCopyInto(out *AMI) { *out = *in if in.Requirements != nil { in, out := &in.Requirements, &out.Requirements - *out = make([]apisv1beta1.NodeSelectorRequirementWithMinValues, len(*in)) + *out = make([]v1.NodeSelectorRequirement, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -320,6 +321,13 @@ func (in *EC2NodeClassStatus) DeepCopyInto(out *EC2NodeClassStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]status.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EC2NodeClassStatus. diff --git a/pkg/batcher/createfleet.go b/pkg/batcher/createfleet.go index 2d7a17557391..6f9ab1221d7b 100644 --- a/pkg/batcher/createfleet.go +++ b/pkg/batcher/createfleet.go @@ -22,7 +22,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" - "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/log" ) type CreateFleetBatcher struct { @@ -70,7 +70,7 @@ func execCreateFleetBatch(ec2api ec2iface.EC2API) BatchExecutor[ec2.CreateFleetI for _, instanceID := range reservation.InstanceIds { requestIdx++ if requestIdx >= len(inputs) { - logging.FromContext(ctx).Errorf("received more instances than requested, ignoring instance %s", aws.StringValue(instanceID)) + log.FromContext(ctx).Error(fmt.Errorf("received more instances than requested, ignoring instance %s", aws.StringValue(instanceID)), "received error while batching") continue } results = append(results, Result[ec2.CreateFleetOutput]{ diff --git a/pkg/batcher/describeinstances.go b/pkg/batcher/describeinstances.go index b0b9df7a4853..961402aa5dcd 100644 --- a/pkg/batcher/describeinstances.go +++ b/pkg/batcher/describeinstances.go @@ -26,7 +26,7 @@ import ( "github.com/mitchellh/hashstructure/v2" "github.com/samber/lo" "k8s.io/apimachinery/pkg/util/sets" - "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/log" ) type DescribeInstancesBatcher struct { @@ -56,7 +56,7 @@ func (b *DescribeInstancesBatcher) DescribeInstances(ctx context.Context, descri func FilterHasher(ctx context.Context, input *ec2.DescribeInstancesInput) uint64 { hash, err := hashstructure.Hash(input.Filters, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}) if err != nil { - logging.FromContext(ctx).Errorf("error hashing") + log.FromContext(ctx).Error(err, "failed hashing input filters") } return hash } diff --git a/pkg/batcher/suite_test.go b/pkg/batcher/suite_test.go index 3f33fb033ec3..523c05c7e213 100644 --- a/pkg/batcher/suite_test.go +++ b/pkg/batcher/suite_test.go @@ -30,7 +30,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) var fakeEC2API *fake.EC2API diff --git a/pkg/batcher/terminateinstances.go b/pkg/batcher/terminateinstances.go index ba3442e02dd6..c1d5d6d49c37 100644 --- a/pkg/batcher/terminateinstances.go +++ b/pkg/batcher/terminateinstances.go @@ -25,7 +25,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/samber/lo" "k8s.io/apimachinery/pkg/util/sets" - "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/log" ) type TerminateInstancesBatcher struct { @@ -68,7 +68,7 @@ func execTerminateInstancesBatch(ec2api ec2iface.EC2API) BatchExecutor[ec2.Termi // We don't care about the error here since we'll break up the batch upon any sort of failure output, err := ec2api.TerminateInstancesWithContext(ctx, firstInput) if err != nil { - logging.FromContext(ctx).Errorf("terminating instances, %s", err) + log.FromContext(ctx).Error(err, "failed terminating instances") } if output == nil { diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 7bb586693f6d..8f8c98d48c7b 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -31,6 +31,13 @@ const ( InstanceTypesAndZonesTTL = 5 * time.Minute // InstanceProfileTTL is the time before we refresh checking instance profile existence at IAM InstanceProfileTTL = 15 * time.Minute + // AvailableIPAddressTTL is time to drop AvailableIPAddress data if it is not updated within the TTL + AvailableIPAddressTTL = 5 * time.Minute + // AvailableIPAddressTTL is time to drop AssociatePublicIPAddressTTL data if it is not updated within the TTL + AssociatePublicIPAddressTTL = 5 * time.Minute + // SSMGetParametersByPathTTL is the time to drop SSM Parameters by path data. This only queries EKS Optimized AMI + // releases, so we should expect this to be updated relatively infrequently. + SSMGetParametersByPathTTL = 24 * time.Hour ) const ( diff --git a/pkg/cache/unavailableofferings.go b/pkg/cache/unavailableofferings.go index bbc2c16b3fb5..e909d4fce161 100644 --- a/pkg/cache/unavailableofferings.go +++ b/pkg/cache/unavailableofferings.go @@ -22,7 +22,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/patrickmn/go-cache" - "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/log" ) // UnavailableOfferings stores any offerings that return ICE (insufficient capacity errors) when @@ -54,12 +54,12 @@ func (u *UnavailableOfferings) IsUnavailable(instanceType, zone, capacityType st // MarkUnavailable communicates recently observed temporary capacity shortages in the provided offerings func (u *UnavailableOfferings) MarkUnavailable(ctx context.Context, unavailableReason, instanceType, zone, capacityType string) { // even if the key is already in the cache, we still need to call Set to extend the cached entry's TTL - logging.FromContext(ctx).With( + log.FromContext(ctx).WithValues( "reason", unavailableReason, "instance-type", instanceType, "zone", zone, "capacity-type", capacityType, - "ttl", UnavailableOfferingsTTL).Debugf("removing offering from offerings") + "ttl", UnavailableOfferingsTTL).V(1).Info("removing offering from offerings") u.cache.SetDefault(u.key(instanceType, zone, capacityType), struct{}{}) atomic.AddUint64(&u.SeqNum, 1) } diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index acea3187e4d1..81ee525ea522 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -21,24 +21,26 @@ import ( "time" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/awslabs/operatorpkg/status" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime/schema" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/log" + coreapis "sigs.k8s.io/karpenter/pkg/apis" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/events" "sigs.k8s.io/karpenter/pkg/scheduling" - "sigs.k8s.io/karpenter/pkg/utils/functional" "sigs.k8s.io/karpenter/pkg/utils/resources" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/pkg/apis" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/utils" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "knative.dev/pkg/logging" "sigs.k8s.io/controller-runtime/pkg/client" cloudproviderevents "github.com/aws/karpenter-provider-aws/pkg/cloudprovider/events" @@ -46,7 +48,6 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/providers/instancetype" "github.com/aws/karpenter-provider-aws/pkg/providers/securitygroup" - "github.com/aws/karpenter-provider-aws/pkg/providers/subnet" "sigs.k8s.io/karpenter/pkg/cloudprovider" ) @@ -61,24 +62,23 @@ type CloudProvider struct { instanceProvider instance.Provider amiProvider amifamily.Provider securityGroupProvider securitygroup.Provider - subnetProvider subnet.Provider } func New(instanceTypeProvider instancetype.Provider, instanceProvider instance.Provider, recorder events.Recorder, - kubeClient client.Client, amiProvider amifamily.Provider, securityGroupProvider securitygroup.Provider, subnetProvider subnet.Provider) *CloudProvider { + kubeClient client.Client, amiProvider amifamily.Provider, securityGroupProvider securitygroup.Provider) *CloudProvider { return &CloudProvider{ instanceTypeProvider: instanceTypeProvider, instanceProvider: instanceProvider, kubeClient: kubeClient, amiProvider: amiProvider, securityGroupProvider: securityGroupProvider, - subnetProvider: subnetProvider, recorder: recorder, } } // Create a NodeClaim given the constraints. -func (c *CloudProvider) Create(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (*corev1beta1.NodeClaim, error) { +// nolint: gocyclo +func (c *CloudProvider) Create(ctx context.Context, nodeClaim *karpv1.NodeClaim) (*karpv1.NodeClaim, error) { nodeClass, err := c.resolveNodeClassFromNodeClaim(ctx, nodeClaim) if err != nil { if errors.IsNotFound(err) { @@ -87,6 +87,27 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *corev1beta1.NodeC // We treat a failure to resolve the NodeClass as an ICE since this means there is no capacity possibilities for this NodeClaim return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("resolving node class, %w", err)) } + + // TODO: Remove this once support for conversion webhooks is dropped + if nodeClass.UbuntuIncompatible() { + return nil, cloudprovider.NewNodeClassNotReadyError(fmt.Errorf("EC2NodeClass %q is incompatible with Karpenter v1, specify your Ubuntu AMIs in your AMISelectorTerms", nodeClass.Name)) + } + // TODO: Remove this after v1 + nodePool, err := utils.ResolveNodePoolFromNodeClaim(ctx, c.kubeClient, nodeClaim) + if err != nil { + return nil, err + } + kubeletHash, err := utils.GetHashKubelet(nodePool, nodeClass) + if err != nil { + return nil, err + } + nodeClassReady := nodeClass.StatusConditions().Get(status.ConditionReady) + if nodeClassReady.IsFalse() { + return nil, cloudprovider.NewNodeClassNotReadyError(fmt.Errorf(nodeClassReady.Message)) + } + if nodeClassReady.IsUnknown() { + return nil, fmt.Errorf("resolving NodeClass readiness, NodeClass is in Ready=Unknown, %s", nodeClassReady.Message) + } instanceTypes, err := c.resolveInstanceTypes(ctx, nodeClaim, nodeClass) if err != nil { return nil, fmt.Errorf("resolving instance types, %w", err) @@ -101,36 +122,41 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *corev1beta1.NodeC instanceType, _ := lo.Find(instanceTypes, func(i *cloudprovider.InstanceType) bool { return i.Name == instance.Type }) - nc := c.instanceToNodeClaim(instance, instanceType) - nc.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash(), - v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, + nc := c.instanceToNodeClaim(instance, instanceType, nodeClass) + nc.Annotations = lo.Assign(nc.Annotations, map[string]string{ + v1.AnnotationKubeletCompatibilityHash: kubeletHash, + v1.AnnotationEC2NodeClassHash: nodeClass.Hash(), + v1.AnnotationEC2NodeClassHashVersion: v1.EC2NodeClassHashVersion, }) return nc, nil } -func (c *CloudProvider) List(ctx context.Context) ([]*corev1beta1.NodeClaim, error) { +func (c *CloudProvider) List(ctx context.Context) ([]*karpv1.NodeClaim, error) { instances, err := c.instanceProvider.List(ctx) if err != nil { return nil, fmt.Errorf("listing instances, %w", err) } - var nodeClaims []*corev1beta1.NodeClaim + var nodeClaims []*karpv1.NodeClaim for _, instance := range instances { instanceType, err := c.resolveInstanceTypeFromInstance(ctx, instance) if err != nil { return nil, fmt.Errorf("resolving instance type, %w", err) } - nodeClaims = append(nodeClaims, c.instanceToNodeClaim(instance, instanceType)) + nc, err := c.resolveNodeClassFromInstance(ctx, instance) + if client.IgnoreNotFound(err) != nil { + return nil, fmt.Errorf("resolving nodeclass, %w", err) + } + nodeClaims = append(nodeClaims, c.instanceToNodeClaim(instance, instanceType, nc)) } return nodeClaims, nil } -func (c *CloudProvider) Get(ctx context.Context, providerID string) (*corev1beta1.NodeClaim, error) { +func (c *CloudProvider) Get(ctx context.Context, providerID string) (*karpv1.NodeClaim, error) { id, err := utils.ParseInstanceID(providerID) if err != nil { return nil, fmt.Errorf("getting instance ID, %w", err) } - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("id", id)) + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("id", id)) instance, err := c.instanceProvider.Get(ctx, id) if err != nil { return nil, fmt.Errorf("getting instance, %w", err) @@ -139,7 +165,11 @@ func (c *CloudProvider) Get(ctx context.Context, providerID string) (*corev1beta if err != nil { return nil, fmt.Errorf("resolving instance type, %w", err) } - return c.instanceToNodeClaim(instance, instanceType), nil + nc, err := c.resolveNodeClassFromInstance(ctx, instance) + if client.IgnoreNotFound(err) != nil { + return nil, fmt.Errorf("resolving nodeclass, %w", err) + } + return c.instanceToNodeClaim(instance, instanceType, nc), nil } func (c *CloudProvider) LivenessProbe(req *http.Request) error { @@ -147,10 +177,7 @@ func (c *CloudProvider) LivenessProbe(req *http.Request) error { } // GetInstanceTypes returns all available InstanceTypes -func (c *CloudProvider) GetInstanceTypes(ctx context.Context, nodePool *corev1beta1.NodePool) ([]*cloudprovider.InstanceType, error) { - if nodePool == nil { - return c.instanceTypeProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, &v1beta1.EC2NodeClass{}) - } +func (c *CloudProvider) GetInstanceTypes(ctx context.Context, nodePool *karpv1.NodePool) ([]*cloudprovider.InstanceType, error) { nodeClass, err := c.resolveNodeClassFromNodePool(ctx, nodePool) if err != nil { if errors.IsNotFound(err) { @@ -161,32 +188,34 @@ func (c *CloudProvider) GetInstanceTypes(ctx context.Context, nodePool *corev1be // as the cause. return nil, fmt.Errorf("resolving node class, %w", err) } + kubeletConfig, err := utils.GetKubletConfigurationWithNodePool(nodePool, nodeClass) + if err != nil { + return nil, fmt.Errorf("resolving kubelet configuration, %w", err) + } // TODO, break this coupling - instanceTypes, err := c.instanceTypeProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + instanceTypes, err := c.instanceTypeProvider.List(ctx, kubeletConfig, nodeClass) if err != nil { return nil, err } return instanceTypes, nil } -func (c *CloudProvider) Delete(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) error { - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("nodeclaim", nodeClaim.Name)) - +func (c *CloudProvider) Delete(ctx context.Context, nodeClaim *karpv1.NodeClaim) error { id, err := utils.ParseInstanceID(nodeClaim.Status.ProviderID) if err != nil { return fmt.Errorf("getting instance ID, %w", err) } - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("id", id)) + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("id", id)) return c.instanceProvider.Delete(ctx, id) } -func (c *CloudProvider) IsDrifted(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (cloudprovider.DriftReason, error) { +func (c *CloudProvider) IsDrifted(ctx context.Context, nodeClaim *karpv1.NodeClaim) (cloudprovider.DriftReason, error) { // Not needed when GetInstanceTypes removes nodepool dependency - nodePoolName, ok := nodeClaim.Labels[corev1beta1.NodePoolLabelKey] + nodePoolName, ok := nodeClaim.Labels[karpv1.NodePoolLabelKey] if !ok { return "", nil } - nodePool := &corev1beta1.NodePool{} + nodePool := &karpv1.NodePool{} if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodePoolName}, nodePool); err != nil { return "", client.IgnoreNotFound(err) } @@ -212,18 +241,12 @@ func (c *CloudProvider) Name() string { return "aws" } -func (c *CloudProvider) GetSupportedNodeClasses() []schema.GroupVersionKind { - return []schema.GroupVersionKind{ - { - Group: v1beta1.SchemeGroupVersion.Group, - Version: v1beta1.SchemeGroupVersion.Version, - Kind: "EC2NodeClass", - }, - } +func (c *CloudProvider) GetSupportedNodeClasses() []status.Object { + return []status.Object{&v1.EC2NodeClass{}} } -func (c *CloudProvider) resolveNodeClassFromNodeClaim(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (*v1beta1.EC2NodeClass, error) { - nodeClass := &v1beta1.EC2NodeClass{} +func (c *CloudProvider) resolveNodeClassFromNodeClaim(ctx context.Context, nodeClaim *karpv1.NodeClaim) (*v1.EC2NodeClass, error) { + nodeClass := &v1.EC2NodeClass{} if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodeClaim.Spec.NodeClassRef.Name}, nodeClass); err != nil { return nil, err } @@ -236,8 +259,8 @@ func (c *CloudProvider) resolveNodeClassFromNodeClaim(ctx context.Context, nodeC return nodeClass, nil } -func (c *CloudProvider) resolveNodeClassFromNodePool(ctx context.Context, nodePool *corev1beta1.NodePool) (*v1beta1.EC2NodeClass, error) { - nodeClass := &v1beta1.EC2NodeClass{} +func (c *CloudProvider) resolveNodeClassFromNodePool(ctx context.Context, nodePool *karpv1.NodePool) (*v1.EC2NodeClass, error) { + nodeClass := &v1.EC2NodeClass{} if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodePool.Spec.Template.Spec.NodeClassRef.Name}, nodeClass); err != nil { return nil, err } @@ -249,8 +272,12 @@ func (c *CloudProvider) resolveNodeClassFromNodePool(ctx context.Context, nodePo return nodeClass, nil } -func (c *CloudProvider) resolveInstanceTypes(ctx context.Context, nodeClaim *corev1beta1.NodeClaim, nodeClass *v1beta1.EC2NodeClass) ([]*cloudprovider.InstanceType, error) { - instanceTypes, err := c.instanceTypeProvider.List(ctx, nodeClaim.Spec.Kubelet, nodeClass) +func (c *CloudProvider) resolveInstanceTypes(ctx context.Context, nodeClaim *karpv1.NodeClaim, nodeClass *v1.EC2NodeClass) ([]*cloudprovider.InstanceType, error) { + kubeletConfig, err := utils.GetKubeletConfigurationWithNodeClaim(nodeClaim, nodeClass) + if err != nil { + return nil, fmt.Errorf("resovling kubelet configuration, %w", err) + } + instanceTypes, err := c.instanceTypeProvider.List(ctx, kubeletConfig, nodeClass) if err != nil { return nil, fmt.Errorf("getting instance types, %w", err) } @@ -279,19 +306,28 @@ func (c *CloudProvider) resolveInstanceTypeFromInstance(ctx context.Context, ins return instanceType, nil } -func (c *CloudProvider) resolveNodePoolFromInstance(ctx context.Context, instance *instance.Instance) (*corev1beta1.NodePool, error) { - if nodePoolName, ok := instance.Tags[corev1beta1.NodePoolLabelKey]; ok { - nodePool := &corev1beta1.NodePool{} +func (c *CloudProvider) resolveNodeClassFromInstance(ctx context.Context, instance *instance.Instance) (*v1.EC2NodeClass, error) { + np, err := c.resolveNodePoolFromInstance(ctx, instance) + if err != nil { + return nil, fmt.Errorf("resolving nodepool, %w", err) + } + return c.resolveNodeClassFromNodePool(ctx, np) +} + +func (c *CloudProvider) resolveNodePoolFromInstance(ctx context.Context, instance *instance.Instance) (*karpv1.NodePool, error) { + if nodePoolName, ok := instance.Tags[karpv1.NodePoolLabelKey]; ok { + nodePool := &karpv1.NodePool{} if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodePoolName}, nodePool); err != nil { return nil, err } return nodePool, nil } - return nil, errors.NewNotFound(schema.GroupResource{Group: corev1beta1.Group, Resource: "nodepools"}, "") + return nil, errors.NewNotFound(schema.GroupResource{Group: coreapis.Group, Resource: "nodepools"}, "") } -func (c *CloudProvider) instanceToNodeClaim(i *instance.Instance, instanceType *cloudprovider.InstanceType) *corev1beta1.NodeClaim { - nodeClaim := &corev1beta1.NodeClaim{} +//nolint:gocyclo +func (c *CloudProvider) instanceToNodeClaim(i *instance.Instance, instanceType *cloudprovider.InstanceType, nodeClass *v1.EC2NodeClass) *karpv1.NodeClaim { + nodeClaim := &karpv1.NodeClaim{} labels := map[string]string{} annotations := map[string]string{} @@ -301,27 +337,35 @@ func (c *CloudProvider) instanceToNodeClaim(i *instance.Instance, instanceType * labels[key] = req.Values()[0] } } - resourceFilter := func(n v1.ResourceName, v resource.Quantity) bool { + resourceFilter := func(n corev1.ResourceName, v resource.Quantity) bool { if resources.IsZero(v) { return false } // The nodeclaim should only advertise an EFA resource if it was requested. EFA network interfaces are only // added to the launch template if they're requested, otherwise the instance is launched with a normal ENI. - if n == v1beta1.ResourceEFA { + if n == v1.ResourceEFA { return i.EFAEnabled } return true } - nodeClaim.Status.Capacity = functional.FilterMap(instanceType.Capacity, resourceFilter) - nodeClaim.Status.Allocatable = functional.FilterMap(instanceType.Allocatable(), resourceFilter) - } - labels[v1.LabelTopologyZone] = i.Zone - labels[corev1beta1.CapacityTypeLabelKey] = i.CapacityType - if v, ok := i.Tags[corev1beta1.NodePoolLabelKey]; ok { - labels[corev1beta1.NodePoolLabelKey] = v + nodeClaim.Status.Capacity = lo.PickBy(instanceType.Capacity, resourceFilter) + nodeClaim.Status.Allocatable = lo.PickBy(instanceType.Allocatable(), resourceFilter) + } + labels[corev1.LabelTopologyZone] = i.Zone + // Attempt to resolve the zoneID from the instance's EC2NodeClass' status condition. + // If the EC2NodeClass is nil, we know we're in the List or Get paths, where we don't care about the zone-id value. + // If we're in the Create path, we've already validated the EC2NodeClass exists. In this case, we resolve the zone-id from the status condition + // both when creating offerings and when adding the label. + if nodeClass != nil { + if subnet, ok := lo.Find(nodeClass.Status.Subnets, func(s v1.Subnet) bool { + return s.Zone == i.Zone + }); ok && subnet.ZoneID != "" { + labels[v1.LabelTopologyZoneID] = subnet.ZoneID + } } - if v, ok := i.Tags[corev1beta1.ManagedByAnnotationKey]; ok { - annotations[corev1beta1.ManagedByAnnotationKey] = v + labels[karpv1.CapacityTypeLabelKey] = i.CapacityType + if v, ok := i.Tags[karpv1.NodePoolLabelKey]; ok { + labels[karpv1.NodePoolLabelKey] = v } nodeClaim.Labels = labels nodeClaim.Annotations = annotations @@ -337,7 +381,7 @@ func (c *CloudProvider) instanceToNodeClaim(i *instance.Instance, instanceType * // newTerminatingNodeClassError returns a NotFound error for handling by func newTerminatingNodeClassError(name string) *errors.StatusError { - qualifiedResource := schema.GroupResource{Group: corev1beta1.Group, Resource: "ec2nodeclasses"} + qualifiedResource := schema.GroupResource{Group: apis.Group, Resource: "ec2nodeclasses"} err := errors.NewNotFound(qualifiedResource, name) err.ErrStatus.Message = fmt.Sprintf("%s %q is terminating, treating as not found", qualifiedResource.String(), name) return err diff --git a/pkg/cloudprovider/drift.go b/pkg/cloudprovider/drift.go index 69dc5aad1783..9fcddddfdf97 100644 --- a/pkg/cloudprovider/drift.go +++ b/pkg/cloudprovider/drift.go @@ -19,16 +19,13 @@ import ( "fmt" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/utils" @@ -41,11 +38,15 @@ const ( NodeClassDrift cloudprovider.DriftReason = "NodeClassDrift" ) -func (c *CloudProvider) isNodeClassDrifted(ctx context.Context, nodeClaim *corev1beta1.NodeClaim, nodePool *corev1beta1.NodePool, nodeClass *v1beta1.EC2NodeClass) (cloudprovider.DriftReason, error) { +func (c *CloudProvider) isNodeClassDrifted(ctx context.Context, nodeClaim *karpv1.NodeClaim, nodePool *karpv1.NodePool, nodeClass *v1.EC2NodeClass) (cloudprovider.DriftReason, error) { // First check if the node class is statically drifted to save on API calls. if drifted := c.areStaticFieldsDrifted(nodeClaim, nodeClass); drifted != "" { return drifted, nil } + kubeletDrifted, err := c.isKubeletConfigurationDrifted(nodeClaim, nodeClass, nodePool) + if err != nil { + return "", err + } instance, err := c.getInstance(ctx, nodeClaim.Status.ProviderID) if err != nil { return "", err @@ -54,40 +55,36 @@ func (c *CloudProvider) isNodeClassDrifted(ctx context.Context, nodeClaim *corev if err != nil { return "", fmt.Errorf("calculating ami drift, %w", err) } - securitygroupDrifted, err := c.areSecurityGroupsDrifted(ctx, instance, nodeClass) + securitygroupDrifted, err := c.areSecurityGroupsDrifted(instance, nodeClass) if err != nil { return "", fmt.Errorf("calculating securitygroup drift, %w", err) } - subnetDrifted, err := c.isSubnetDrifted(ctx, instance, nodeClass) + subnetDrifted, err := c.isSubnetDrifted(instance, nodeClass) if err != nil { return "", fmt.Errorf("calculating subnet drift, %w", err) } - drifted := lo.FindOrElse([]cloudprovider.DriftReason{amiDrifted, securitygroupDrifted, subnetDrifted}, "", func(i cloudprovider.DriftReason) bool { + drifted := lo.FindOrElse([]cloudprovider.DriftReason{amiDrifted, securitygroupDrifted, subnetDrifted, kubeletDrifted}, "", func(i cloudprovider.DriftReason) bool { return string(i) != "" }) return drifted, nil } -func (c *CloudProvider) isAMIDrifted(ctx context.Context, nodeClaim *corev1beta1.NodeClaim, nodePool *corev1beta1.NodePool, - instance *instance.Instance, nodeClass *v1beta1.EC2NodeClass) (cloudprovider.DriftReason, error) { +func (c *CloudProvider) isAMIDrifted(ctx context.Context, nodeClaim *karpv1.NodeClaim, nodePool *karpv1.NodePool, + instance *instance.Instance, nodeClass *v1.EC2NodeClass) (cloudprovider.DriftReason, error) { instanceTypes, err := c.GetInstanceTypes(ctx, nodePool) if err != nil { return "", fmt.Errorf("getting instanceTypes, %w", err) } nodeInstanceType, found := lo.Find(instanceTypes, func(instType *cloudprovider.InstanceType) bool { - return instType.Name == nodeClaim.Labels[v1.LabelInstanceTypeStable] + return instType.Name == nodeClaim.Labels[corev1.LabelInstanceTypeStable] }) if !found { - return "", fmt.Errorf(`finding node instance type "%s"`, nodeClaim.Labels[v1.LabelInstanceTypeStable]) - } - amis, err := c.amiProvider.Get(ctx, nodeClass, &amifamily.Options{}) - if err != nil { - return "", fmt.Errorf("getting amis, %w", err) + return "", fmt.Errorf(`finding node instance type "%s"`, nodeClaim.Labels[corev1.LabelInstanceTypeStable]) } - if len(amis) == 0 { + if len(nodeClass.Status.AMIs) == 0 { return "", fmt.Errorf("no amis exist given constraints") } - mappedAMIs := amis.MapToInstanceTypes([]*cloudprovider.InstanceType{nodeInstanceType}) + mappedAMIs := amifamily.MapToInstanceTypes([]*cloudprovider.InstanceType{nodeInstanceType}, nodeClass.Status.AMIs) if !lo.Contains(lo.Keys(mappedAMIs), instance.ImageID) { return AMIDrift, nil } @@ -96,18 +93,14 @@ func (c *CloudProvider) isAMIDrifted(ctx context.Context, nodeClaim *corev1beta1 // Checks if the security groups are drifted, by comparing the subnet returned from the subnetProvider // to the ec2 instance subnets -func (c *CloudProvider) isSubnetDrifted(ctx context.Context, instance *instance.Instance, nodeClass *v1beta1.EC2NodeClass) (cloudprovider.DriftReason, error) { - subnets, err := c.subnetProvider.List(ctx, nodeClass) - if err != nil { - return "", err - } +func (c *CloudProvider) isSubnetDrifted(instance *instance.Instance, nodeClass *v1.EC2NodeClass) (cloudprovider.DriftReason, error) { // subnets need to be found to check for drift - if len(subnets) == 0 { + if len(nodeClass.Status.Subnets) == 0 { return "", fmt.Errorf("no subnets are discovered") } - _, found := lo.Find(subnets, func(subnet *ec2.Subnet) bool { - return aws.StringValue(subnet.SubnetId) == instance.SubnetID + _, found := lo.Find(nodeClass.Status.Subnets, func(subnet v1.Subnet) bool { + return subnet.ID == instance.SubnetID }) if !found { @@ -118,14 +111,10 @@ func (c *CloudProvider) isSubnetDrifted(ctx context.Context, instance *instance. // Checks if the security groups are drifted, by comparing the security groups returned from the SecurityGroupProvider // to the ec2 instance security groups -func (c *CloudProvider) areSecurityGroupsDrifted(ctx context.Context, ec2Instance *instance.Instance, nodeClass *v1beta1.EC2NodeClass) (cloudprovider.DriftReason, error) { - securitygroup, err := c.securityGroupProvider.List(ctx, nodeClass) - if err != nil { - return "", err - } - securityGroupIds := sets.New(lo.Map(securitygroup, func(sg *ec2.SecurityGroup, _ int) string { return aws.StringValue(sg.GroupId) })...) +func (c *CloudProvider) areSecurityGroupsDrifted(ec2Instance *instance.Instance, nodeClass *v1.EC2NodeClass) (cloudprovider.DriftReason, error) { + securityGroupIds := sets.New(lo.Map(nodeClass.Status.SecurityGroups, func(sg v1.SecurityGroup, _ int) string { return sg.ID })...) if len(securityGroupIds) == 0 { - return "", fmt.Errorf("no security groups are discovered") + return "", fmt.Errorf("no security groups are present in the status") } if !securityGroupIds.Equal(sets.New(ec2Instance.SecurityGroupIDs...)) { @@ -134,11 +123,11 @@ func (c *CloudProvider) areSecurityGroupsDrifted(ctx context.Context, ec2Instanc return "", nil } -func (c *CloudProvider) areStaticFieldsDrifted(nodeClaim *corev1beta1.NodeClaim, nodeClass *v1beta1.EC2NodeClass) cloudprovider.DriftReason { - nodeClassHash, foundNodeClassHash := nodeClass.Annotations[v1beta1.AnnotationEC2NodeClassHash] - nodeClassHashVersion, foundNodeClassHashVersion := nodeClass.Annotations[v1beta1.AnnotationEC2NodeClassHashVersion] - nodeClaimHash, foundNodeClaimHash := nodeClaim.Annotations[v1beta1.AnnotationEC2NodeClassHash] - nodeClaimHashVersion, foundNodeClaimHashVersion := nodeClaim.Annotations[v1beta1.AnnotationEC2NodeClassHashVersion] +func (c *CloudProvider) areStaticFieldsDrifted(nodeClaim *karpv1.NodeClaim, nodeClass *v1.EC2NodeClass) cloudprovider.DriftReason { + nodeClassHash, foundNodeClassHash := nodeClass.Annotations[v1.AnnotationEC2NodeClassHash] + nodeClassHashVersion, foundNodeClassHashVersion := nodeClass.Annotations[v1.AnnotationEC2NodeClassHashVersion] + nodeClaimHash, foundNodeClaimHash := nodeClaim.Annotations[v1.AnnotationEC2NodeClassHash] + nodeClaimHashVersion, foundNodeClaimHashVersion := nodeClaim.Annotations[v1.AnnotationEC2NodeClassHashVersion] if !foundNodeClassHash || !foundNodeClaimHash || !foundNodeClassHashVersion || !foundNodeClaimHashVersion { return "" @@ -150,6 +139,27 @@ func (c *CloudProvider) areStaticFieldsDrifted(nodeClaim *corev1beta1.NodeClaim, return lo.Ternary(nodeClassHash != nodeClaimHash, NodeClassDrift, "") } +// Remove once v1beta1 is dropped +func (c *CloudProvider) isKubeletConfigurationDrifted(nodeClaim *karpv1.NodeClaim, nodeClass *v1.EC2NodeClass, nodePool *karpv1.NodePool) (cloudprovider.DriftReason, error) { + kubeletHash, err := utils.GetHashKubelet(nodePool, nodeClass) + if err != nil { + return "", err + } + nodeClaimKubeletHash, foundNodeClaimKubeletHash := nodeClaim.Annotations[v1.AnnotationKubeletCompatibilityHash] + nodeClassHashVersion, foundNodeClassHashVersion := nodeClass.Annotations[v1.AnnotationEC2NodeClassHashVersion] + nodeClaimHashVersion, foundNodeClaimHashVersion := nodeClaim.Annotations[v1.AnnotationEC2NodeClassHashVersion] + + if !foundNodeClaimKubeletHash || !foundNodeClaimHashVersion || !foundNodeClassHashVersion { + return "", nil + } + + // validate that the hash version for the EC2NodeClass is the same as the NodeClaim before evaluating for static drift + if nodeClassHashVersion != nodeClaimHashVersion { + return "", nil + } + return lo.Ternary(kubeletHash != nodeClaimKubeletHash, NodeClassDrift, ""), nil +} + func (c *CloudProvider) getInstance(ctx context.Context, providerID string) (*instance.Instance, error) { // Get InstanceID to fetch from EC2 instanceID, err := utils.ParseInstanceID(providerID) diff --git a/pkg/cloudprovider/events/events.go b/pkg/cloudprovider/events/events.go index f571bc2baed2..e5c167f031c0 100644 --- a/pkg/cloudprovider/events/events.go +++ b/pkg/cloudprovider/events/events.go @@ -15,25 +15,25 @@ limitations under the License. package events import ( - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + v1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/events" ) -func NodePoolFailedToResolveNodeClass(nodePool *v1beta1.NodePool) events.Event { +func NodePoolFailedToResolveNodeClass(nodePool *v1.NodePool) events.Event { return events.Event{ InvolvedObject: nodePool, - Type: v1.EventTypeWarning, + Type: corev1.EventTypeWarning, Message: "Failed resolving NodeClass", DedupeValues: []string{string(nodePool.UID)}, } } -func NodeClaimFailedToResolveNodeClass(nodeClaim *v1beta1.NodeClaim) events.Event { +func NodeClaimFailedToResolveNodeClass(nodeClaim *v1.NodeClaim) events.Event { return events.Event{ InvolvedObject: nodeClaim, - Type: v1.EventTypeWarning, + Type: corev1.EventTypeWarning, Message: "Failed resolving NodeClass", DedupeValues: []string{string(nodeClaim.UID)}, } diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 556767b6d109..88ef02bfd879 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -22,40 +22,43 @@ import ( "testing" "time" - v1 "k8s.io/api/core/v1" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + + "github.com/awslabs/operatorpkg/object" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" - clock "k8s.io/utils/clock/testing" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/ssm" + opstatus "github.com/awslabs/operatorpkg/status" "github.com/imdario/mergo" "github.com/samber/lo" "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" + "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/test" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - corecloudproivder "sigs.k8s.io/karpenter/pkg/cloudprovider" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/controllers/provisioning" "sigs.k8s.io/karpenter/pkg/controllers/state" "sigs.k8s.io/karpenter/pkg/events" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" + "sigs.k8s.io/karpenter/pkg/scheduling" coretest "sigs.k8s.io/karpenter/pkg/test" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) var ctx context.Context @@ -75,7 +78,7 @@ func TestAWS(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) ctx, stop = context.WithCancel(ctx) @@ -83,8 +86,8 @@ var _ = BeforeSuite(func() { fakeClock = clock.NewFakeClock(time.Now()) recorder = events.NewRecorder(&record.FakeRecorder{}) cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, recorder, - env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider, awsEnv.SubnetProvider) - cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) + env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider) + cluster = state.NewCluster(fakeClock, env.Client) prov = provisioning.NewProvisioner(env.Client, recorder, cloudProvider, cluster) }) @@ -109,50 +112,126 @@ var _ = AfterEach(func() { }) var _ = Describe("CloudProvider", func() { - var nodeClass *v1beta1.EC2NodeClass - var nodePool *corev1beta1.NodePool - var nodeClaim *corev1beta1.NodeClaim + var nodeClass *v1.EC2NodeClass + var nodePool *karpv1.NodePool + var nodeClaim *karpv1.NodeClaim var _ = BeforeEach(func() { - nodeClass = test.EC2NodeClass() - nodePool = coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nodeClass = test.EC2NodeClass( + v1.EC2NodeClass{ + Status: v1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + SecurityGroups: []v1.SecurityGroup{ + { + ID: "sg-test1", + Name: "securityGroup-test1", + }, + { + ID: "sg-test2", + Name: "securityGroup-test2", + }, + { + ID: "sg-test3", + Name: "securityGroup-test3", }, - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeOnDemand}}}, + }, + Subnets: []v1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + ZoneID: "tstz1-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + ZoneID: "tstz1-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + ZoneID: "tstz1-1c", + }, + }, + }, + }, + ) + nodeClass.StatusConditions().SetTrue(opstatus.ConditionReady) + nodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: karpv1.CapacityTypeLabelKey, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.CapacityTypeOnDemand}}}, }, }, }, }, }) - nodeClaim = coretest.NodeClaim(corev1beta1.NodeClaim{ + nodeClaim = coretest.NodeClaim(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{corev1beta1.NodePoolLabelKey: nodePool.Name}, + Labels: map[string]string{karpv1.NodePoolLabelKey: nodePool.Name}, }, - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, + }, + }, }, }, }) + _, err := awsEnv.SubnetProvider.List(ctx, nodeClass) // Hydrate the subnet cache + Expect(err).To(BeNil()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) + }) + // TODO: remove after v1 + It("should fail instance creation if NodeClass has ubuntu incompatible annotation", func() { + nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1.AnnotationUbuntuCompatibilityKey: v1.AnnotationUbuntuCompatibilityIncompatible}) + ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim) + _, err := cloudProvider.Create(ctx, nodeClaim) + Expect(corecloudprovider.IsNodeClassNotReadyError(err)).To(BeTrue()) + }) + It("should not proceed with instance creation if NodeClass is unknown", func() { + nodeClass.StatusConditions().SetUnknown(opstatus.ConditionReady) + ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim) + _, err := cloudProvider.Create(ctx, nodeClaim) + Expect(err).To(HaveOccurred()) + Expect(corecloudprovider.IsNodeClassNotReadyError(err)).To(BeFalse()) + }) + It("should return NodeClassNotReady error on creation if NodeClass is not ready", func() { + nodeClass.StatusConditions().SetFalse(opstatus.ConditionReady, "NodeClassNotReady", "NodeClass not ready") + ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim) + _, err := cloudProvider.Create(ctx, nodeClaim) + Expect(err).To(HaveOccurred()) + Expect(corecloudprovider.IsNodeClassNotReadyError(err)).To(BeTrue()) }) It("should return an ICE error when there are no instance types to launch", func() { // Specify no instance types and expect to receive a capacity error - nodeClaim.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodeClaim.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"test-instance-type"}, }, }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim) cloudProviderNodeClaim, err := cloudProvider.Create(ctx, nodeClaim) - Expect(corecloudproivder.IsInsufficientCapacityError(err)).To(BeTrue()) + Expect(corecloudprovider.IsInsufficientCapacityError(err)).To(BeTrue()) Expect(cloudProviderNodeClaim).To(BeNil()) }) It("should set ImageID in the status field of the nodeClaim", func() { @@ -162,12 +241,35 @@ var _ = Describe("CloudProvider", func() { Expect(cloudProviderNodeClaim).ToNot(BeNil()) Expect(cloudProviderNodeClaim.Status.ImageID).ToNot(BeEmpty()) }) + It("should return availability zone ID as a label on the nodeClaim", func() { + ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim) + cloudProviderNodeClaim, err := cloudProvider.Create(ctx, nodeClaim) + Expect(err).ToNot(HaveOccurred()) + Expect(cloudProviderNodeClaim).ToNot(BeNil()) + zone, ok := cloudProviderNodeClaim.GetLabels()[corev1.LabelTopologyZone] + Expect(ok).To(BeTrue()) + zoneID, ok := cloudProviderNodeClaim.GetLabels()[v1.LabelTopologyZoneID] + Expect(ok).To(BeTrue()) + subnet, ok := lo.Find(nodeClass.Status.Subnets, func(s v1.Subnet) bool { + return s.Zone == zone + }) + Expect(ok).To(BeTrue()) + Expect(zoneID).To(Equal(subnet.ZoneID)) + }) + It("should expect a strict set of annotation keys", func() { + ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim) + cloudProviderNodeClaim, err := cloudProvider.Create(ctx, nodeClaim) + Expect(err).To(BeNil()) + Expect(cloudProviderNodeClaim).ToNot(BeNil()) + Expect(len(lo.Keys(cloudProviderNodeClaim.Annotations))).To(BeNumerically("==", 3)) + Expect(lo.Keys(cloudProviderNodeClaim.Annotations)).To(ContainElements(v1.AnnotationKubeletCompatibilityHash, v1.AnnotationEC2NodeClassHash, v1.AnnotationEC2NodeClassHashVersion)) + }) It("should return NodeClass Hash on the nodeClaim", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim) cloudProviderNodeClaim, err := cloudProvider.Create(ctx, nodeClaim) Expect(err).To(BeNil()) Expect(cloudProviderNodeClaim).ToNot(BeNil()) - _, ok := cloudProviderNodeClaim.ObjectMeta.Annotations[v1beta1.AnnotationEC2NodeClassHash] + _, ok := cloudProviderNodeClaim.ObjectMeta.Annotations[v1.AnnotationEC2NodeClassHash] Expect(ok).To(BeTrue()) }) It("should return NodeClass Hash Version on the nodeClaim", func() { @@ -175,9 +277,9 @@ var _ = Describe("CloudProvider", func() { cloudProviderNodeClaim, err := cloudProvider.Create(ctx, nodeClaim) Expect(err).To(BeNil()) Expect(cloudProviderNodeClaim).ToNot(BeNil()) - v, ok := cloudProviderNodeClaim.ObjectMeta.Annotations[v1beta1.AnnotationEC2NodeClassHashVersion] + v, ok := cloudProviderNodeClaim.ObjectMeta.Annotations[v1.AnnotationEC2NodeClassHashVersion] Expect(ok).To(BeTrue()) - Expect(v).To(Equal(v1beta1.EC2NodeClassHashVersion)) + Expect(v).To(Equal(v1.EC2NodeClassHashVersion)) }) Context("EC2 Context", func() { contextID := "context-1234" @@ -230,29 +332,33 @@ var _ = Describe("CloudProvider", func() { }, }, }) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) instanceNames := lo.Map(instances, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType }) // Define NodePool that has minValues on instance-type requirement. - nodePool = coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeSpot}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: instanceNames, }, MinValues: lo.ToPtr(2), @@ -268,14 +374,14 @@ var _ = Describe("CloudProvider", func() { // 2 pods are created with resources such that both fit together only in one of the 2 InstanceTypes created above. pod1 := coretest.UnschedulablePod( coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("0.9")}, + ResourceRequirements: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0.9")}, }, }) pod2 := coretest.UnschedulablePod( coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("0.9")}, + ResourceRequirements: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0.9")}, }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod1, pod2) @@ -324,29 +430,33 @@ var _ = Describe("CloudProvider", func() { }, }, }) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) instanceNames := lo.Map(instances, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType }) // Define NodePool that has minValues on instance-type requirement. - nodePool = coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpExists, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpExists, }, MinValues: lo.ToPtr(2), }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: instanceNames, }, MinValues: lo.ToPtr(1), @@ -362,14 +472,14 @@ var _ = Describe("CloudProvider", func() { // 2 pods are created with resources such that both fit together only in one of the 2 InstanceTypes created above. pod1 := coretest.UnschedulablePod( coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("0.9")}, + ResourceRequirements: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0.9")}, }, }) pod2 := coretest.UnschedulablePod( coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("0.9")}, + ResourceRequirements: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0.9")}, }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod1, pod2) @@ -425,31 +535,35 @@ var _ = Describe("CloudProvider", func() { }, }, }) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) instanceNames := lo.Map(uniqInstanceTypes, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType }) // Define NodePool that has minValues in multiple requirements. - nodePool = coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: instanceNames, }, // consider at least 2 unique instance types MinValues: lo.ToPtr(2), }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceFamily, + Operator: corev1.NodeSelectorOpIn, Values: instanceFamilies.UnsortedList(), }, // consider at least 3 unique instance families @@ -464,14 +578,14 @@ var _ = Describe("CloudProvider", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod1 := coretest.UnschedulablePod( coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("0.9")}, + ResourceRequirements: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0.9")}, }, }) pod2 := coretest.UnschedulablePod( coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("0.9")}, + ResourceRequirements: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0.9")}, }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod1, pod2) @@ -500,7 +614,7 @@ var _ = Describe("CloudProvider", func() { Context("NodeClaim Drift", func() { var armAMIID, amdAMIID string var validSecurityGroup string - var selectedInstanceType *corecloudproivder.InstanceType + var selectedInstanceType *corecloudprovider.InstanceType var instance *ec2.Instance var validSubnet1 string var validSubnet2 string @@ -509,9 +623,6 @@ var _ = Describe("CloudProvider", func() { validSecurityGroup = fake.SecurityGroupID() validSubnet1 = fake.SubnetID() validSubnet2 = fake.SubnetID() - awsEnv.SSMAPI.GetParameterOutput = &ssm.GetParameterOutput{ - Parameter: &ssm.Parameter{Value: aws.String(armAMIID)}, - } awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ Images: []*ec2.Image{ { @@ -578,14 +689,52 @@ var _ = Describe("CloudProvider", func() { }, }, }) + nodeClass.Status = v1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + Subnets: []v1.Subnet{ + { + ID: validSubnet1, + Zone: "zone-1", + }, + { + ID: validSubnet2, + Zone: "zone-2", + }, + }, + SecurityGroups: []v1.SecurityGroup{ + { + ID: validSecurityGroup, + }, + }, + AMIs: []v1.AMI{ + { + ID: armAMIID, + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureArm64}}, + }, + }, + { + ID: amdAMIID, + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureAmd64}}, + }, + }, + }, + } ExpectApplied(ctx, env.Client, nodePool, nodeClass) instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, nodePool) Expect(err).ToNot(HaveOccurred()) - selectedInstanceType = instanceTypes[0] + var ok bool + selectedInstanceType, ok = lo.Find(instanceTypes, func(i *corecloudprovider.InstanceType) bool { + return i.Requirements.Compatible(scheduling.NewLabelRequirements(map[string]string{ + corev1.LabelArchStable: karpv1.ArchitectureAmd64, + })) == nil + }) + Expect(ok).To(BeTrue()) // Create the instance we want returned from the EC2 API instance = &ec2.Instance{ - ImageId: aws.String(armAMIID), + ImageId: aws.String(amdAMIID), InstanceType: aws.String(selectedInstanceType.Name), SubnetId: aws.String(validSubnet1), SpotInstanceRequestId: aws.String(coretest.RandomName()), @@ -602,15 +751,15 @@ var _ = Describe("CloudProvider", func() { Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{instance}}}, }) nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash(), - v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, + v1.AnnotationEC2NodeClassHash: nodeClass.Hash(), + v1.AnnotationEC2NodeClassHashVersion: v1.EC2NodeClassHashVersion, }) nodeClaim.Status.ProviderID = fake.ProviderID(lo.FromPtr(instance.InstanceId)) nodeClaim.Annotations = lo.Assign(nodeClaim.Annotations, map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash(), - v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, + v1.AnnotationEC2NodeClassHash: nodeClass.Hash(), + v1.AnnotationEC2NodeClassHashVersion: v1.EC2NodeClassHashVersion, }) - nodeClaim.Labels = lo.Assign(nodeClaim.Labels, map[string]string{v1.LabelInstanceTypeStable: selectedInstanceType.Name}) + nodeClaim.Labels = lo.Assign(nodeClaim.Labels, map[string]string{corev1.LabelInstanceTypeStable: selectedInstanceType.Name}) }) It("should not fail if NodeClass does not exist", func() { ExpectDeleted(ctx, env.Client, nodeClass) @@ -638,7 +787,7 @@ var _ = Describe("CloudProvider", func() { instance.SecurityGroups = []*ec2.GroupIdentifier{{GroupId: aws.String(fake.SecurityGroupID())}} // Assign a fake hash nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "abcdefghijkl", + v1.AnnotationEC2NodeClassHash: "abcdefghijkl", }) ExpectApplied(ctx, env.Client, nodeClass) isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) @@ -653,7 +802,8 @@ var _ = Describe("CloudProvider", func() { }) It("should return an error if subnets are empty", func() { awsEnv.SubnetCache.Flush() - awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{}}) + nodeClass.Status.Subnets = []v1.Subnet{} + ExpectApplied(ctx, env.Client, nodeClass) _, err := cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).To(HaveOccurred()) }) @@ -663,7 +813,8 @@ var _ = Describe("CloudProvider", func() { Expect(isDrifted).To(BeEmpty()) }) It("should return an error if the security groups are empty", func() { - awsEnv.EC2API.DescribeSecurityGroupsOutput.Set(&ec2.DescribeSecurityGroupsOutput{SecurityGroups: []*ec2.SecurityGroup{}}) + nodeClass.Status.SecurityGroups = []v1.SecurityGroup{} + ExpectApplied(ctx, env.Client, nodeClass) // Instance is a reference to what we return in the GetInstances call instance.SecurityGroups = []*ec2.GroupIdentifier{{GroupId: aws.String(fake.SecurityGroupID())}} _, err := cloudProvider.IsDrifted(ctx, nodeClaim) @@ -684,18 +835,17 @@ var _ = Describe("CloudProvider", func() { Expect(isDrifted).To(Equal(cloudprovider.SecurityGroupDrift)) }) It("should return drifted if more security groups are present than instance security groups then discovered from nodeclass", func() { - awsEnv.EC2API.DescribeSecurityGroupsOutput.Set(&ec2.DescribeSecurityGroupsOutput{ - SecurityGroups: []*ec2.SecurityGroup{ - { - GroupId: aws.String(validSecurityGroup), - GroupName: aws.String("test-securitygroup"), - }, - { - GroupId: aws.String(fake.SecurityGroupID()), - GroupName: aws.String("test-securitygroup"), - }, + nodeClass.Status.SecurityGroups = []v1.SecurityGroup{ + { + ID: validSecurityGroup, + Name: "test-securitygroup", }, - }) + { + ID: fake.SecurityGroupID(), + Name: "test-securitygroup", + }, + } + ExpectApplied(ctx, env.Client, nodeClass) isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).ToNot(HaveOccurred()) Expect(isDrifted).To(Equal(cloudprovider.SecurityGroupDrift)) @@ -706,12 +856,12 @@ var _ = Describe("CloudProvider", func() { Expect(isDrifted).To(BeEmpty()) }) It("should error if the NodeClaim doesn't have the instance-type label", func() { - delete(nodeClaim.Labels, v1.LabelInstanceTypeStable) + delete(nodeClaim.Labels, corev1.LabelInstanceTypeStable) _, err := cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).To(HaveOccurred()) }) It("should error drift if NodeClaim doesn't have provider id", func() { - nodeClaim.Status = corev1beta1.NodeClaimStatus{} + nodeClaim.Status = karpv1.NodeClaimStatus{} isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).To(HaveOccurred()) Expect(isDrifted).To(BeEmpty()) @@ -724,7 +874,17 @@ var _ = Describe("CloudProvider", func() { Expect(err).To(HaveOccurred()) }) It("should return drifted if the AMI no longer matches the existing NodeClaims instance type", func() { - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: amdAMIID}} + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: amdAMIID}} + nodeClass.Status.AMIs = []v1.AMI{ + { + ID: amdAMIID, + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureAmd64}}, + }, + }, + } + instance.ImageId = aws.String(armAMIID) ExpectApplied(ctx, env.Client, nodeClass) isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).ToNot(HaveOccurred()) @@ -732,9 +892,15 @@ var _ = Describe("CloudProvider", func() { }) Context("Static Drift Detection", func() { BeforeEach(func() { - nodeClass = &v1beta1.EC2NodeClass{ + armRequirements := []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureArm64}}, + } + amdRequirements := []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureAmd64}}, + } + nodeClass = &v1.EC2NodeClass{ ObjectMeta: nodeClass.ObjectMeta, - Spec: v1beta1.EC2NodeClassSpec{ + Spec: v1.EC2NodeClassSpec{ SubnetSelectorTerms: nodeClass.Spec.SubnetSelectorTerms, SecurityGroupSelectorTerms: nodeClass.Spec.SecurityGroupSelectorTerms, Role: nodeClass.Spec.Role, @@ -742,21 +908,23 @@ var _ = Describe("CloudProvider", func() { Tags: map[string]string{ "fakeKey": "fakeValue", }, - Context: lo.ToPtr("fake-context"), - DetailedMonitoring: lo.ToPtr(false), - AMIFamily: lo.ToPtr(v1beta1.AMIFamilyAL2023), + Context: lo.ToPtr("fake-context"), + DetailedMonitoring: lo.ToPtr(false), + AMISelectorTerms: []v1.AMISelectorTerm{{ + Alias: "al2023@latest", + }}, AssociatePublicIPAddress: lo.ToPtr(false), - MetadataOptions: &v1beta1.MetadataOptions{ + MetadataOptions: &v1.MetadataOptions{ HTTPEndpoint: lo.ToPtr("disabled"), HTTPProtocolIPv6: lo.ToPtr("disabled"), HTTPPutResponseHopLimit: lo.ToPtr(int64(1)), HTTPTokens: lo.ToPtr("optional"), }, - BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + BlockDeviceMappings: []*v1.BlockDeviceMapping{ { DeviceName: lo.ToPtr("fakeName"), RootVolume: false, - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ DeleteOnTermination: lo.ToPtr(false), Encrypted: lo.ToPtr(false), IOPS: lo.ToPtr(int64(0)), @@ -769,51 +937,78 @@ var _ = Describe("CloudProvider", func() { }, }, }, + Status: v1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + Subnets: []v1.Subnet{ + { + ID: validSubnet1, + Zone: "zone-1", + }, + { + ID: validSubnet2, + Zone: "zone-2", + }, + }, + SecurityGroups: []v1.SecurityGroup{ + { + ID: validSecurityGroup, + }, + }, + AMIs: []v1.AMI{ + { + ID: armAMIID, + Requirements: armRequirements, + }, + { + ID: amdAMIID, + Requirements: amdRequirements, + }, + }, + }, } - nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) - nodeClaim.Annotations = lo.Assign(nodeClaim.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) + nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) + nodeClaim.Annotations = lo.Assign(nodeClaim.Annotations, map[string]string{v1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) }) DescribeTable("should return drifted if a statically drifted EC2NodeClass.Spec field is updated", - func(changes v1beta1.EC2NodeClass) { + func(changes v1.EC2NodeClass) { ExpectApplied(ctx, env.Client, nodePool, nodeClass) isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).NotTo(HaveOccurred()) Expect(isDrifted).To(BeEmpty()) Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride, mergo.WithSliceDeepCopy)).To(Succeed()) - nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) + nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) ExpectApplied(ctx, env.Client, nodeClass) isDrifted, err = cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).NotTo(HaveOccurred()) Expect(isDrifted).To(Equal(cloudprovider.NodeClassDrift)) }, - Entry("UserData", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{UserData: lo.ToPtr("userdata-test-2")}}), - Entry("Tags", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), - Entry("Context", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: lo.ToPtr("context-2")}}), - Entry("DetailedMonitoring", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), - Entry("AMIFamily", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: lo.ToPtr(v1beta1.AMIFamilyBottlerocket)}}), - Entry("InstanceStorePolicy", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{InstanceStorePolicy: lo.ToPtr(v1beta1.InstanceStorePolicyRAID0)}}), - Entry("AssociatePublicIPAddress", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AssociatePublicIPAddress: lo.ToPtr(true)}}), - Entry("MetadataOptions HTTPEndpoint", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: lo.ToPtr("enabled")}}}), - Entry("MetadataOptions HTTPProtocolIPv6", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPProtocolIPv6: lo.ToPtr("enabled")}}}), - Entry("MetadataOptions HTTPPutResponseHopLimit", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPPutResponseHopLimit: lo.ToPtr(int64(10))}}}), - Entry("MetadataOptions HTTPTokens", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPTokens: lo.ToPtr("required")}}}), - Entry("BlockDeviceMapping DeviceName", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: lo.ToPtr("map-device-test-3")}}}}), - Entry("BlockDeviceMapping RootVolume", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{RootVolume: true}}}}), - Entry("BlockDeviceMapping DeleteOnTermination", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{DeleteOnTermination: lo.ToPtr(true)}}}}}), - Entry("BlockDeviceMapping Encrypted", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{Encrypted: lo.ToPtr(true)}}}}}), - Entry("BlockDeviceMapping IOPS", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{IOPS: lo.ToPtr(int64(10))}}}}}), - Entry("BlockDeviceMapping KMSKeyID", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{KMSKeyID: lo.ToPtr("test")}}}}}), - Entry("BlockDeviceMapping SnapshotID", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{SnapshotID: lo.ToPtr("test")}}}}}), - Entry("BlockDeviceMapping Throughput", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{Throughput: lo.ToPtr(int64(10))}}}}}), - Entry("BlockDeviceMapping VolumeType", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{VolumeType: lo.ToPtr("io1")}}}}}), + Entry("UserData", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{UserData: lo.ToPtr("userdata-test-2")}}), + Entry("Tags", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + Entry("Context", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Context: lo.ToPtr("context-2")}}), + Entry("DetailedMonitoring", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), + Entry("InstanceStorePolicy", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{InstanceStorePolicy: lo.ToPtr(v1.InstanceStorePolicyRAID0)}}), + Entry("AssociatePublicIPAddress", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{AssociatePublicIPAddress: lo.ToPtr(true)}}), + Entry("MetadataOptions HTTPEndpoint", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPEndpoint: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPProtocolIPv6", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPProtocolIPv6: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPPutResponseHopLimit", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPPutResponseHopLimit: lo.ToPtr(int64(10))}}}), + Entry("MetadataOptions HTTPTokens", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPTokens: lo.ToPtr("required")}}}), + Entry("BlockDeviceMapping DeviceName", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{DeviceName: lo.ToPtr("map-device-test-3")}}}}), + Entry("BlockDeviceMapping RootVolume", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{RootVolume: true}}}}), + Entry("BlockDeviceMapping DeleteOnTermination", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{DeleteOnTermination: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping Encrypted", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{Encrypted: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping IOPS", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{IOPS: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping KMSKeyID", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{KMSKeyID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping SnapshotID", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{SnapshotID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping Throughput", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{Throughput: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping VolumeType", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{VolumeType: lo.ToPtr("io1")}}}}}), ) // We create a separate test for updating blockDeviceMapping volumeSize, since resource.Quantity is a struct, and mergo.WithSliceDeepCopy // doesn't work well with unexported fields, like the ones that are present in resource.Quantity It("should return drifted when updating blockDeviceMapping volumeSize", func() { nodeClass.Spec.BlockDeviceMappings[0].EBS.VolumeSize = resource.NewScaledQuantity(10, resource.Giga) - nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) + nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) ExpectApplied(ctx, env.Client, nodeClass) isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) @@ -821,27 +1016,30 @@ var _ = Describe("CloudProvider", func() { Expect(isDrifted).To(Equal(cloudprovider.NodeClassDrift)) }) DescribeTable("should not return drifted if dynamic fields are updated", - func(changes v1beta1.EC2NodeClass) { + func(changes v1.EC2NodeClass) { ExpectApplied(ctx, env.Client, nodePool, nodeClass) isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).NotTo(HaveOccurred()) Expect(isDrifted).To(BeEmpty()) Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride)) - nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) + nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) ExpectApplied(ctx, env.Client, nodeClass) isDrifted, err = cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).NotTo(HaveOccurred()) Expect(isDrifted).To(BeEmpty()) }, - Entry("AMI Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMISelectorTerms: []v1beta1.AMISelectorTerm{{Tags: map[string]string{"ami-key-1": "ami-value-1"}}}}}), - Entry("Subnet Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{{Tags: map[string]string{"sn-key-1": "sn-value-1"}}}}}), - Entry("SecurityGroup Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{{Tags: map[string]string{"sg-key": "sg-value"}}}}}), + Entry("AMI Drift", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{ + AMIFamily: lo.ToPtr(v1.AMIFamilyAL2023), + AMISelectorTerms: []v1.AMISelectorTerm{{Tags: map[string]string{"ami-key-1": "ami-value-1"}}}, + }}), + Entry("Subnet Drift", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{SubnetSelectorTerms: []v1.SubnetSelectorTerm{{Tags: map[string]string{"sn-key-1": "sn-value-1"}}}}}), + Entry("SecurityGroup Drift", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{{Tags: map[string]string{"sg-key": "sg-value"}}}}}), ) It("should not return drifted if karpenter.k8s.aws/ec2nodeclass-hash annotation is not present on the NodeClaim", func() { nodeClaim.Annotations = map[string]string{ - v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, + v1.AnnotationEC2NodeClassHashVersion: v1.EC2NodeClassHashVersion, } nodeClass.Spec.Tags = map[string]string{ "Test Key": "Test Value", @@ -853,12 +1051,12 @@ var _ = Describe("CloudProvider", func() { }) It("should not return drifted if the NodeClaim's karpenter.k8s.aws/ec2nodeclass-hash-version annotation does not match the EC2NodeClass's", func() { nodeClass.ObjectMeta.Annotations = map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "test-hash-111111", - v1beta1.AnnotationEC2NodeClassHashVersion: "test-hash-version-1", + v1.AnnotationEC2NodeClassHash: "test-hash-111111", + v1.AnnotationEC2NodeClassHashVersion: "test-hash-version-1", } nodeClaim.ObjectMeta.Annotations = map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "test-hash-222222", - v1beta1.AnnotationEC2NodeClassHashVersion: "test-hash-version-2", + v1.AnnotationEC2NodeClassHash: "test-hash-222222", + v1.AnnotationEC2NodeClassHashVersion: "test-hash-version-2", } ExpectApplied(ctx, env.Client, nodePool, nodeClass) isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) @@ -867,11 +1065,11 @@ var _ = Describe("CloudProvider", func() { }) It("should not return drifted if karpenter.k8s.aws/ec2nodeclass-hash-version annotation is not present on the NodeClass", func() { nodeClass.ObjectMeta.Annotations = map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "test-hash-111111", + v1.AnnotationEC2NodeClassHash: "test-hash-111111", } nodeClaim.ObjectMeta.Annotations = map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "test-hash-222222", - v1beta1.AnnotationEC2NodeClassHashVersion: "test-hash-version-2", + v1.AnnotationEC2NodeClassHash: "test-hash-222222", + v1.AnnotationEC2NodeClassHashVersion: "test-hash-version-2", } // should trigger drift nodeClass.Spec.Tags = map[string]string{ @@ -884,11 +1082,11 @@ var _ = Describe("CloudProvider", func() { }) It("should not return drifted if karpenter.k8s.aws/ec2nodeclass-hash-version annotation is not present on the NodeClaim", func() { nodeClass.ObjectMeta.Annotations = map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "test-hash-111111", - v1beta1.AnnotationEC2NodeClassHashVersion: "test-hash-version-1", + v1.AnnotationEC2NodeClassHash: "test-hash-111111", + v1.AnnotationEC2NodeClassHashVersion: "test-hash-version-1", } nodeClaim.ObjectMeta.Annotations = map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "test-hash-222222", + v1.AnnotationEC2NodeClassHash: "test-hash-222222", } // should trigger drift nodeClass.Spec.Tags = map[string]string{ @@ -907,7 +1105,7 @@ var _ = Describe("CloudProvider", func() { It("should default to the cluster's subnets", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod( - coretest.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: corev1beta1.ArchitectureAmd64}}) + coretest.PodOptions{NodeSelector: map[string]string{corev1.LabelArchStable: karpv1.ArchitectureAmd64}}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) @@ -930,37 +1128,46 @@ var _ = Describe("CloudProvider", func() { Expect(foundNonGPULT).To(BeTrue()) }) It("should launch instances into subnet with the most available IP addresses", func() { + awsEnv.SubnetCache.Flush() awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ - {SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(10), + {SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(10), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}}, - {SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(100), + {SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(100), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}}, }}) + controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) + ExpectObjectReconciled(ctx, env.Client, controller, nodeClass) + pod := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{corev1.LabelTopologyZone: "test-zone-1a"}}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-2")) }) It("should launch instances into subnet with the most available IP addresses in-between cache refreshes", func() { + awsEnv.SubnetCache.Flush() awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ - {SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(10), + {SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(10), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}}, - {SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(11), + {SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(11), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}}, }}) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: aws.Int32(1)} + controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + MaxPods: aws.Int32(1), + } ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod1 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) - pod2 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, controller, nodeClass) + pod1 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{corev1.LabelTopologyZone: "test-zone-1a"}}) + pod2 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{corev1.LabelTopologyZone: "test-zone-1a"}}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod1, pod2) ExpectScheduled(ctx, env.Client, pod1) ExpectScheduled(ctx, env.Client, pod2) createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-2")) // Provision for another pod that should now use the other subnet since we've consumed some from the first launch. - pod3 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) + pod3 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{corev1.LabelTopologyZone: "test-zone-1a"}}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod3) ExpectScheduled(ctx, env.Client, pod3) createFleetInput = awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() @@ -971,7 +1178,7 @@ var _ = Describe("CloudProvider", func() { {SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(10), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}}, }}) - pod1 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) + pod1 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{corev1.LabelTopologyZone: "test-zone-1a"}}) ExpectApplied(ctx, env.Client, nodePool, nodeClass, pod1) awsEnv.EC2API.CreateFleetBehavior.Error.Set(fmt.Errorf("CreateFleet synthetic error")) bindings := ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod1) @@ -979,80 +1186,96 @@ var _ = Describe("CloudProvider", func() { }) It("should launch instances into subnets that are excluded by another NodePool", func() { awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ - {SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(10), + {SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(10), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}}, - {SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1b"), AvailableIpAddressCount: aws.Int64(100), + {SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1b"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(100), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}}, }}) - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{{Tags: map[string]string{"Name": "test-subnet-1"}}} + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{{Tags: map[string]string{"Name": "test-subnet-1"}}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) + controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) + ExpectObjectReconciled(ctx, env.Client, controller, nodeClass) podSubnet1 := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, podSubnet1) ExpectScheduled(ctx, env.Client, podSubnet1) createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-1")) - nodeClass2 := test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ - SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ + nodeClass2 := test.EC2NodeClass(v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ { Tags: map[string]string{"Name": "test-subnet-2"}, }, }, - SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, }, + Status: v1.EC2NodeClassStatus{ + AMIs: nodeClass.Status.AMIs, + SecurityGroups: []v1.SecurityGroup{ + { + ID: "sg-test1", + }, + }, + }, }) - nodePool2 := coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass2.Name, + nodePool2 := coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass2).Group, + Kind: object.GVK(nodeClass2).Kind, + Name: nodeClass2.Name, }, }, }, }, }) ExpectApplied(ctx, env.Client, nodePool2, nodeClass2) - podSubnet2 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{corev1beta1.NodePoolLabelKey: nodePool2.Name}}) + ExpectObjectReconciled(ctx, env.Client, controller, nodeClass2) + podSubnet2 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{karpv1.NodePoolLabelKey: nodePool2.Name}}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, podSubnet2) ExpectScheduled(ctx, env.Client, podSubnet2) createFleetInput = awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-2")) }) It("should launch instances with an alternate NodePool when a NodeClass selects 0 subnets, security groups, or amis", func() { - misconfiguredNodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ + misconfiguredNodeClass := test.EC2NodeClass(v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ // select nothing! - SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ { Tags: map[string]string{"Name": "nothing"}, }, }, // select nothing! - SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"Name": "nothing"}, }, }, + AMIFamily: lo.ToPtr(v1.AMIFamilyCustom), // select nothing! - AMISelectorTerms: []v1beta1.AMISelectorTerm{ + AMISelectorTerms: []v1.AMISelectorTerm{ { Tags: map[string]string{"Name": "nothing"}, }, }, }, }) - nodePool2 := coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: misconfiguredNodeClass.Name, + nodePool2 := coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(misconfiguredNodeClass).Group, + Kind: object.GVK(misconfiguredNodeClass).Kind, + Name: misconfiguredNodeClass.Name, }, }, }, @@ -1066,27 +1289,27 @@ var _ = Describe("CloudProvider", func() { }) Context("EFA", func() { It("should include vpc.amazonaws.com/efa on a nodeclaim if it requests it", func() { - nodeClaim.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodeClaim.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"dl1.24xlarge"}, }, }, } - nodeClaim.Spec.Resources.Requests = v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("1")} + nodeClaim.Spec.Resources.Requests = corev1.ResourceList{v1.ResourceEFA: resource.MustParse("1")} ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim) cloudProviderNodeClaim, err := cloudProvider.Create(ctx, nodeClaim) Expect(err).To(BeNil()) - Expect(lo.Keys(cloudProviderNodeClaim.Status.Allocatable)).To(ContainElement(v1beta1.ResourceEFA)) + Expect(lo.Keys(cloudProviderNodeClaim.Status.Allocatable)).To(ContainElement(v1.ResourceEFA)) }) It("shouldn't include vpc.amazonaws.com/efa on a nodeclaim if it doesn't request it", func() { - nodeClaim.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodeClaim.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"dl1.24xlarge"}, }, }, @@ -1094,7 +1317,7 @@ var _ = Describe("CloudProvider", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim) cloudProviderNodeClaim, err := cloudProvider.Create(ctx, nodeClaim) Expect(err).To(BeNil()) - Expect(lo.Keys(cloudProviderNodeClaim.Status.Allocatable)).ToNot(ContainElement(v1beta1.ResourceEFA)) + Expect(lo.Keys(cloudProviderNodeClaim.Status.Allocatable)).ToNot(ContainElement(v1.ResourceEFA)) }) }) }) diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index eb8f606e72a7..c45f00938299 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -17,12 +17,17 @@ package controllers import ( "context" + "github.com/awslabs/operatorpkg/controller" + "github.com/awslabs/operatorpkg/status" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/karpenter/pkg/cloudprovider" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" nodeclasshash "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/hash" nodeclassstatus "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status" nodeclasstermination "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/termination" - controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/pricing" + controllersinstancetype "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/instancetype" + controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/pricing" "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" "github.com/aws/aws-sdk-go/aws/session" @@ -32,7 +37,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/karpenter/pkg/events" - "sigs.k8s.io/karpenter/pkg/operator/controller" "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption" @@ -42,16 +46,17 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" + "github.com/aws/karpenter-provider-aws/pkg/providers/instancetype" "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" "github.com/aws/karpenter-provider-aws/pkg/providers/securitygroup" "github.com/aws/karpenter-provider-aws/pkg/providers/sqs" "github.com/aws/karpenter-provider-aws/pkg/providers/subnet" ) -func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, kubeClient client.Client, recorder events.Recorder, +func NewControllers(ctx context.Context, mgr manager.Manager, sess *session.Session, clk clock.Clock, kubeClient client.Client, recorder events.Recorder, unavailableOfferings *cache.UnavailableOfferings, cloudProvider cloudprovider.CloudProvider, subnetProvider subnet.Provider, securityGroupProvider securitygroup.Provider, instanceProfileProvider instanceprofile.Provider, instanceProvider instance.Provider, - pricingProvider pricing.Provider, amiProvider amifamily.Provider, launchTemplateProvider launchtemplate.Provider) []controller.Controller { + pricingProvider pricing.Provider, amiProvider amifamily.Provider, launchTemplateProvider launchtemplate.Provider, instanceTypeProvider instancetype.Provider) []controller.Controller { controllers := []controller.Controller{ nodeclasshash.NewController(kubeClient), @@ -60,6 +65,8 @@ func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider), nodeclaimtagging.NewController(kubeClient, instanceProvider), controllerspricing.NewController(pricingProvider), + controllersinstancetype.NewController(instanceTypeProvider), + status.NewController[*v1.EC2NodeClass](kubeClient, mgr.GetEventRecorderFor("karpenter")), } if options.FromContext(ctx).InterruptionQueue != "" { sqsapi := servicesqs.New(sess) diff --git a/pkg/controllers/interruption/controller.go b/pkg/controllers/interruption/controller.go index 589db848f75b..731e3a741ac2 100644 --- a/pkg/controllers/interruption/controller.go +++ b/pkg/controllers/interruption/controller.go @@ -19,31 +19,33 @@ import ( "fmt" "time" - sqsapi "github.com/aws/aws-sdk-go/service/sqs" "github.com/prometheus/client_golang/prometheus" - "github.com/samber/lo" + "sigs.k8s.io/karpenter/pkg/metrics" + + sqsapi "github.com/aws/aws-sdk-go/service/sqs" + "github.com/awslabs/operatorpkg/singleton" "go.uber.org/multierr" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" "k8s.io/utils/clock" - "knative.dev/pkg/logging" + controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/karpenter/pkg/metrics" + "sigs.k8s.io/karpenter/pkg/operator/injection" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/utils/pretty" "github.com/aws/karpenter-provider-aws/pkg/cache" interruptionevents "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/events" "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/messages" - "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/messages/statechange" "github.com/aws/karpenter-provider-aws/pkg/providers/sqs" "github.com/aws/karpenter-provider-aws/pkg/utils" "sigs.k8s.io/karpenter/pkg/events" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" ) type Action string @@ -80,17 +82,18 @@ func NewController(kubeClient client.Client, clk clock.Clock, recorder events.Re } } -func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("queue", c.sqsProvider.Name())) +func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) { + ctx = injection.WithControllerName(ctx, "interruption") + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("queue", c.sqsProvider.Name())) if c.cm.HasChanged(c.sqsProvider.Name(), nil) { - logging.FromContext(ctx).Debugf("watching interruption queue") + log.FromContext(ctx).V(1).Info("watching interruption queue") } sqsMessages, err := c.sqsProvider.GetSQSMessages(ctx) if err != nil { return reconcile.Result{}, fmt.Errorf("getting messages from queue, %w", err) } if len(sqsMessages) == 0 { - return reconcile.Result{}, nil + return reconcile.Result{RequeueAfter: singleton.RequeueImmediately}, nil } nodeClaimInstanceIDMap, err := c.makeNodeClaimInstanceIDMap(ctx) if err != nil { @@ -105,7 +108,7 @@ func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconc msg, e := c.parseMessage(sqsMessages[i]) if e != nil { // If we fail to parse, then we should delete the message but still log the error - logging.FromContext(ctx).Errorf("parsing message, %v", e) + log.FromContext(ctx).Error(err, "failed parsing interruption message") errs[i] = c.deleteMessage(ctx, sqsMessages[i]) return } @@ -118,15 +121,14 @@ func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconc if err = multierr.Combine(errs...); err != nil { return reconcile.Result{}, err } - return reconcile.Result{}, nil + return reconcile.Result{RequeueAfter: singleton.RequeueImmediately}, nil } -func (c *Controller) Name() string { - return "interruption" -} - -func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { - return corecontroller.NewSingletonManagedBy(m) +func (c *Controller) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("interruption"). + WatchesRawSource(singleton.Source()). + Complete(singleton.AsReconciler(c)) } // parseMessage parses the passed SQS message into an internal Message interface @@ -143,10 +145,10 @@ func (c *Controller) parseMessage(raw *sqsapi.Message) (messages.Message, error) } // handleMessage takes an action against every node involved in the message that is owned by a NodePool -func (c *Controller) handleMessage(ctx context.Context, nodeClaimInstanceIDMap map[string]*v1beta1.NodeClaim, - nodeInstanceIDMap map[string]*v1.Node, msg messages.Message) (err error) { +func (c *Controller) handleMessage(ctx context.Context, nodeClaimInstanceIDMap map[string]*karpv1.NodeClaim, + nodeInstanceIDMap map[string]*corev1.Node, msg messages.Message) (err error) { - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("messageKind", msg.Kind())) + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("messageKind", msg.Kind())) receivedMessages.WithLabelValues(string(msg.Kind())).Inc() if msg.Kind() == messages.NoOpKind { @@ -179,51 +181,50 @@ func (c *Controller) deleteMessage(ctx context.Context, msg *sqsapi.Message) err } // handleNodeClaim retrieves the action for the message and then performs the appropriate action against the node -func (c *Controller) handleNodeClaim(ctx context.Context, msg messages.Message, nodeClaim *v1beta1.NodeClaim, node *v1.Node) error { +func (c *Controller) handleNodeClaim(ctx context.Context, msg messages.Message, nodeClaim *karpv1.NodeClaim, node *corev1.Node) error { action := actionForMessage(msg) - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("nodeclaim", nodeClaim.Name, "action", string(action))) + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("NodeClaim", klog.KRef("", nodeClaim.Name), "action", string(action))) if node != nil { - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("node", node.Name)) + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("Node", klog.KRef("", node.Name))) } // Record metric and event for this action c.notifyForMessage(msg, nodeClaim, node) - actionsPerformed.WithLabelValues(string(action)).Inc() // Mark the offering as unavailable in the ICE cache since we got a spot interruption warning if msg.Kind() == messages.SpotInterruptionKind { - zone := nodeClaim.Labels[v1.LabelTopologyZone] - instanceType := nodeClaim.Labels[v1.LabelInstanceTypeStable] + zone := nodeClaim.Labels[corev1.LabelTopologyZone] + instanceType := nodeClaim.Labels[corev1.LabelInstanceTypeStable] if zone != "" && instanceType != "" { - c.unavailableOfferingsCache.MarkUnavailable(ctx, string(msg.Kind()), instanceType, zone, v1beta1.CapacityTypeSpot) + c.unavailableOfferingsCache.MarkUnavailable(ctx, string(msg.Kind()), instanceType, zone, karpv1.CapacityTypeSpot) } } if action != NoAction { - return c.deleteNodeClaim(ctx, nodeClaim, node) + return c.deleteNodeClaim(ctx, msg, nodeClaim, node) } return nil } // deleteNodeClaim removes the NodeClaim from the api-server -func (c *Controller) deleteNodeClaim(ctx context.Context, nodeClaim *v1beta1.NodeClaim, node *v1.Node) error { +func (c *Controller) deleteNodeClaim(ctx context.Context, msg messages.Message, nodeClaim *karpv1.NodeClaim, node *corev1.Node) error { if !nodeClaim.DeletionTimestamp.IsZero() { return nil } if err := c.kubeClient.Delete(ctx, nodeClaim); err != nil { return client.IgnoreNotFound(fmt.Errorf("deleting the node on interruption message, %w", err)) } - logging.FromContext(ctx).Infof("initiating delete from interruption message") + log.FromContext(ctx).Info("initiating delete from interruption message") c.recorder.Publish(interruptionevents.TerminatingOnInterruption(node, nodeClaim)...) - metrics.NodeClaimsTerminatedCounter.With(prometheus.Labels{ - metrics.ReasonLabel: terminationReasonLabel, - metrics.NodePoolLabel: nodeClaim.Labels[v1beta1.NodePoolLabelKey], - metrics.CapacityTypeLabel: nodeClaim.Labels[v1beta1.CapacityTypeLabelKey], + metrics.NodeClaimsDisruptedTotal.With(prometheus.Labels{ + metrics.ReasonLabel: string(msg.Kind()), + metrics.NodePoolLabel: nodeClaim.Labels[karpv1.NodePoolLabelKey], + metrics.CapacityTypeLabel: nodeClaim.Labels[karpv1.CapacityTypeLabelKey], }).Inc() return nil } // notifyForMessage publishes the relevant alert based on the message kind -func (c *Controller) notifyForMessage(msg messages.Message, nodeClaim *v1beta1.NodeClaim, n *v1.Node) { +func (c *Controller) notifyForMessage(msg messages.Message, nodeClaim *karpv1.NodeClaim, n *corev1.Node) { switch msg.Kind() { case messages.RebalanceRecommendationKind: c.recorder.Publish(interruptionevents.RebalanceRecommendation(n, nodeClaim)...) @@ -234,13 +235,11 @@ func (c *Controller) notifyForMessage(msg messages.Message, nodeClaim *v1beta1.N case messages.SpotInterruptionKind: c.recorder.Publish(interruptionevents.SpotInterrupted(n, nodeClaim)...) - case messages.StateChangeKind: - typed := msg.(statechange.Message) - if lo.Contains([]string{"stopping", "stopped"}, typed.Detail.State) { - c.recorder.Publish(interruptionevents.Stopping(n, nodeClaim)...) - } else { - c.recorder.Publish(interruptionevents.Terminating(n, nodeClaim)...) - } + case messages.InstanceStoppedKind: + c.recorder.Publish(interruptionevents.Stopping(n, nodeClaim)...) + + case messages.InstanceTerminatedKind: + c.recorder.Publish(interruptionevents.Terminating(n, nodeClaim)...) default: } @@ -248,9 +247,9 @@ func (c *Controller) notifyForMessage(msg messages.Message, nodeClaim *v1beta1.N // makeNodeClaimInstanceIDMap builds a map between the instance id that is stored in the // NodeClaim .status.providerID and the NodeClaim -func (c *Controller) makeNodeClaimInstanceIDMap(ctx context.Context) (map[string]*v1beta1.NodeClaim, error) { - m := map[string]*v1beta1.NodeClaim{} - nodeClaimList := &v1beta1.NodeClaimList{} +func (c *Controller) makeNodeClaimInstanceIDMap(ctx context.Context) (map[string]*karpv1.NodeClaim, error) { + m := map[string]*karpv1.NodeClaim{} + nodeClaimList := &karpv1.NodeClaimList{} if err := c.kubeClient.List(ctx, nodeClaimList); err != nil { return nil, err } @@ -269,9 +268,9 @@ func (c *Controller) makeNodeClaimInstanceIDMap(ctx context.Context) (map[string // makeNodeInstanceIDMap builds a map between the instance id that is stored in the // node .spec.providerID and the node -func (c *Controller) makeNodeInstanceIDMap(ctx context.Context) (map[string]*v1.Node, error) { - m := map[string]*v1.Node{} - nodeList := &v1.NodeList{} +func (c *Controller) makeNodeInstanceIDMap(ctx context.Context) (map[string]*corev1.Node, error) { + m := map[string]*corev1.Node{} + nodeList := &corev1.NodeList{} if err := c.kubeClient.List(ctx, nodeList); err != nil { return nil, fmt.Errorf("listing nodes, %w", err) } @@ -290,7 +289,7 @@ func (c *Controller) makeNodeInstanceIDMap(ctx context.Context) (map[string]*v1. func actionForMessage(msg messages.Message) Action { switch msg.Kind() { - case messages.ScheduledChangeKind, messages.SpotInterruptionKind, messages.StateChangeKind: + case messages.ScheduledChangeKind, messages.SpotInterruptionKind, messages.InstanceStoppedKind, messages.InstanceTerminatedKind: return CordonAndDrain default: return NoAction diff --git a/pkg/controllers/interruption/events/events.go b/pkg/controllers/interruption/events/events.go index 90146e8ff7fa..fdcecabd0be0 100644 --- a/pkg/controllers/interruption/events/events.go +++ b/pkg/controllers/interruption/events/events.go @@ -15,16 +15,16 @@ limitations under the License. package events import ( - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/events" ) -func SpotInterrupted(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events.Event) { +func SpotInterrupted(node *corev1.Node, nodeClaim *karpv1.NodeClaim) (evts []events.Event) { evts = append(evts, events.Event{ InvolvedObject: nodeClaim, - Type: v1.EventTypeWarning, + Type: corev1.EventTypeWarning, Reason: "SpotInterrupted", Message: "Spot interruption warning was triggered", DedupeValues: []string{string(nodeClaim.UID)}, @@ -32,7 +32,7 @@ func SpotInterrupted(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events if node != nil { evts = append(evts, events.Event{ InvolvedObject: node, - Type: v1.EventTypeWarning, + Type: corev1.EventTypeWarning, Reason: "SpotInterrupted", Message: "Spot interruption warning was triggered", DedupeValues: []string{string(node.UID)}, @@ -41,10 +41,10 @@ func SpotInterrupted(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events return evts } -func RebalanceRecommendation(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events.Event) { +func RebalanceRecommendation(node *corev1.Node, nodeClaim *karpv1.NodeClaim) (evts []events.Event) { evts = append(evts, events.Event{ InvolvedObject: nodeClaim, - Type: v1.EventTypeNormal, + Type: corev1.EventTypeNormal, Reason: "SpotRebalanceRecommendation", Message: "Spot rebalance recommendation was triggered", DedupeValues: []string{string(nodeClaim.UID)}, @@ -52,7 +52,7 @@ func RebalanceRecommendation(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts if node != nil { evts = append(evts, events.Event{ InvolvedObject: node, - Type: v1.EventTypeNormal, + Type: corev1.EventTypeNormal, Reason: "SpotRebalanceRecommendation", Message: "Spot rebalance recommendation was triggered", DedupeValues: []string{string(node.UID)}, @@ -61,10 +61,10 @@ func RebalanceRecommendation(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts return evts } -func Stopping(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events.Event) { +func Stopping(node *corev1.Node, nodeClaim *karpv1.NodeClaim) (evts []events.Event) { evts = append(evts, events.Event{ InvolvedObject: nodeClaim, - Type: v1.EventTypeWarning, + Type: corev1.EventTypeWarning, Reason: "InstanceStopping", Message: "Instance is stopping", DedupeValues: []string{string(nodeClaim.UID)}, @@ -72,7 +72,7 @@ func Stopping(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events.Event) if node != nil { evts = append(evts, events.Event{ InvolvedObject: node, - Type: v1.EventTypeWarning, + Type: corev1.EventTypeWarning, Reason: "InstanceStopping", Message: "Instance is stopping", DedupeValues: []string{string(node.UID)}, @@ -81,10 +81,10 @@ func Stopping(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events.Event) return evts } -func Terminating(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events.Event) { +func Terminating(node *corev1.Node, nodeClaim *karpv1.NodeClaim) (evts []events.Event) { evts = append(evts, events.Event{ InvolvedObject: nodeClaim, - Type: v1.EventTypeWarning, + Type: corev1.EventTypeWarning, Reason: "InstanceTerminating", Message: "Instance is terminating", DedupeValues: []string{string(nodeClaim.UID)}, @@ -92,7 +92,7 @@ func Terminating(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events.Eve if node != nil { evts = append(evts, events.Event{ InvolvedObject: node, - Type: v1.EventTypeWarning, + Type: corev1.EventTypeWarning, Reason: "InstanceTerminating", Message: "Instance is terminating", DedupeValues: []string{string(node.UID)}, @@ -101,10 +101,10 @@ func Terminating(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events.Eve return evts } -func Unhealthy(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events.Event) { +func Unhealthy(node *corev1.Node, nodeClaim *karpv1.NodeClaim) (evts []events.Event) { evts = append(evts, events.Event{ InvolvedObject: nodeClaim, - Type: v1.EventTypeWarning, + Type: corev1.EventTypeWarning, Reason: "InstanceUnhealthy", Message: "An unhealthy warning was triggered for the instance", DedupeValues: []string{string(nodeClaim.UID)}, @@ -112,7 +112,7 @@ func Unhealthy(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events.Event if node != nil { evts = append(evts, events.Event{ InvolvedObject: node, - Type: v1.EventTypeWarning, + Type: corev1.EventTypeWarning, Reason: "InstanceUnhealthy", Message: "An unhealthy warning was triggered for the instance", DedupeValues: []string{string(node.UID)}, @@ -121,10 +121,10 @@ func Unhealthy(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events.Event return evts } -func TerminatingOnInterruption(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evts []events.Event) { +func TerminatingOnInterruption(node *corev1.Node, nodeClaim *karpv1.NodeClaim) (evts []events.Event) { evts = append(evts, events.Event{ InvolvedObject: nodeClaim, - Type: v1.EventTypeWarning, + Type: corev1.EventTypeWarning, Reason: "TerminatingOnInterruption", Message: "Interruption triggered termination for the NodeClaim", DedupeValues: []string{string(nodeClaim.UID)}, @@ -132,7 +132,7 @@ func TerminatingOnInterruption(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (evt if node != nil { evts = append(evts, events.Event{ InvolvedObject: node, - Type: v1.EventTypeWarning, + Type: corev1.EventTypeWarning, Reason: "TerminatingOnInterruption", Message: "Interruption triggered termination for the Node", DedupeValues: []string{string(node.UID)}, diff --git a/pkg/controllers/interruption/interruption_benchmark_test.go b/pkg/controllers/interruption/interruption_benchmark_test.go index 6a9a591db8de..534dcdc0b89f 100644 --- a/pkg/controllers/interruption/interruption_benchmark_test.go +++ b/pkg/controllers/interruption/interruption_benchmark_test.go @@ -32,19 +32,18 @@ import ( "github.com/aws/aws-sdk-go/aws/session" servicesqs "github.com/aws/aws-sdk-go/service/sqs" "github.com/aws/aws-sdk-go/service/sqs/sqsiface" + "github.com/go-logr/zapr" "github.com/samber/lo" "go.uber.org/multierr" "go.uber.org/zap" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/workqueue" clock "k8s.io/utils/clock/testing" - "knative.dev/pkg/logging" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - - "sigs.k8s.io/karpenter/pkg/operator/scheme" + "sigs.k8s.io/controller-runtime/pkg/log" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" awscache "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption" @@ -78,7 +77,7 @@ func BenchmarkNotification100(b *testing.B) { //nolint:gocyclo func benchmarkNotificationController(b *testing.B, messageCount int) { - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("message-count", messageCount)) + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("message-count", messageCount)) fakeClock = &clock.FakeClock{} ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ @@ -86,7 +85,7 @@ func benchmarkNotificationController(b *testing.B, messageCount int) { IsolatedVPC: lo.ToPtr(true), InterruptionQueue: lo.ToPtr("test-cluster"), })) - env = coretest.NewEnvironment(scheme.Scheme) + env = coretest.NewEnvironment() // Stop the coretest environment after the coretest completes defer func() { if err := retry.Do(func() error { @@ -118,20 +117,20 @@ func benchmarkNotificationController(b *testing.B, messageCount int) { interruptionController := interruption.NewController(env.Client, fakeClock, recorder, providers.sqsProvider, unavailableOfferingsCache) messages, nodes := makeDiverseMessagesAndNodes(messageCount) - logging.FromContext(ctx).Infof("provisioning nodes") + log.FromContext(ctx).Info("provisioning nodes") if err := provisionNodes(ctx, env.Client, nodes); err != nil { b.Fatalf("provisioning nodes, %v", err) } - logging.FromContext(ctx).Infof("completed provisioning nodes") + log.FromContext(ctx).Info("completed provisioning nodes") - logging.FromContext(ctx).Infof("provisioning messages into the SQS Queue") + log.FromContext(ctx).Info("provisioning messages into the SQS Queue") if err := providers.provisionMessages(ctx, messages...); err != nil { b.Fatalf("provisioning messages, %v", err) } - logging.FromContext(ctx).Infof("completed provisioning messages into the SQS Queue") + log.FromContext(ctx).Info("completed provisioning messages into the SQS Queue") m, err := controllerruntime.NewManager(env.Config, controllerruntime.Options{ - BaseContext: func() context.Context { return logging.WithLogger(ctx, zap.NewNop().Sugar()) }, + BaseContext: func() context.Context { return log.IntoContext(ctx, zapr.NewLogger(zap.NewNop())) }, }) if err != nil { b.Fatalf("creating manager, %v", err) @@ -146,7 +145,7 @@ func benchmarkNotificationController(b *testing.B, messageCount int) { start := time.Now() managerErr := make(chan error) go func() { - logging.FromContext(ctx).Infof("starting controller manager") + log.FromContext(ctx).Info("starting controller manager") managerErr <- m.Start(ctx) }() @@ -225,7 +224,7 @@ func (p *providerSet) monitorMessagesProcessed(ctx context.Context, eventRecorde eventRecorder.Calls(events.Unhealthy(coretest.Node(), coretest.NodeClaim())[0].Reason) + eventRecorder.Calls(events.RebalanceRecommendation(coretest.Node(), coretest.NodeClaim())[0].Reason) + eventRecorder.Calls(events.SpotInterrupted(coretest.Node(), coretest.NodeClaim())[0].Reason) - logging.FromContext(ctx).With("processed-message-count", totalProcessed).Infof("processed messages from the queue") + log.FromContext(ctx).WithValues("processed-message-count", totalProcessed).Info("processed messages from the queue") time.Sleep(time.Second) } close(done) @@ -233,7 +232,7 @@ func (p *providerSet) monitorMessagesProcessed(ctx context.Context, eventRecorde return done } -func provisionNodes(ctx context.Context, kubeClient client.Client, nodes []*v1.Node) error { +func provisionNodes(ctx context.Context, kubeClient client.Client, nodes []*corev1.Node) error { errs := make([]error, len(nodes)) workqueue.ParallelizeUntil(ctx, 20, len(nodes), func(i int) { if err := retry.Do(func() error { @@ -245,9 +244,9 @@ func provisionNodes(ctx context.Context, kubeClient client.Client, nodes []*v1.N return multierr.Combine(errs...) } -func makeDiverseMessagesAndNodes(count int) ([]interface{}, []*v1.Node) { +func makeDiverseMessagesAndNodes(count int) ([]interface{}, []*corev1.Node) { var messages []interface{} - var nodes []*v1.Node + var nodes []*corev1.Node newMessages, newNodes := makeScheduledChangeMessagesAndNodes(count / 3) messages = append(messages, newMessages...) @@ -266,16 +265,16 @@ func makeDiverseMessagesAndNodes(count int) ([]interface{}, []*v1.Node) { return messages, nodes } -func makeScheduledChangeMessagesAndNodes(count int) ([]interface{}, []*v1.Node) { +func makeScheduledChangeMessagesAndNodes(count int) ([]interface{}, []*corev1.Node) { var msgs []interface{} - var nodes []*v1.Node + var nodes []*corev1.Node for i := 0; i < count; i++ { instanceID := fake.InstanceID() msgs = append(msgs, scheduledChangeMessage(instanceID)) nodes = append(nodes, coretest.Node(coretest.NodeOptions{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - v1beta1.NodePoolLabelKey: "default", + karpv1.NodePoolLabelKey: "default", }, }, ProviderID: fake.ProviderID(instanceID), @@ -284,9 +283,9 @@ func makeScheduledChangeMessagesAndNodes(count int) ([]interface{}, []*v1.Node) return msgs, nodes } -func makeStateChangeMessagesAndNodes(count int, states []string) ([]interface{}, []*v1.Node) { +func makeStateChangeMessagesAndNodes(count int, states []string) ([]interface{}, []*corev1.Node) { var msgs []interface{} - var nodes []*v1.Node + var nodes []*corev1.Node for i := 0; i < count; i++ { state := states[r.Intn(len(states))] instanceID := fake.InstanceID() @@ -294,7 +293,7 @@ func makeStateChangeMessagesAndNodes(count int, states []string) ([]interface{}, nodes = append(nodes, coretest.Node(coretest.NodeOptions{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - v1beta1.NodePoolLabelKey: "default", + karpv1.NodePoolLabelKey: "default", }, }, ProviderID: fake.ProviderID(instanceID), @@ -303,16 +302,16 @@ func makeStateChangeMessagesAndNodes(count int, states []string) ([]interface{}, return msgs, nodes } -func makeSpotInterruptionMessagesAndNodes(count int) ([]interface{}, []*v1.Node) { +func makeSpotInterruptionMessagesAndNodes(count int) ([]interface{}, []*corev1.Node) { var msgs []interface{} - var nodes []*v1.Node + var nodes []*corev1.Node for i := 0; i < count; i++ { instanceID := fake.InstanceID() msgs = append(msgs, spotInterruptionMessage(instanceID)) nodes = append(nodes, coretest.Node(coretest.NodeOptions{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - v1beta1.NodePoolLabelKey: "default", + karpv1.NodePoolLabelKey: "default", }, }, ProviderID: fake.ProviderID(instanceID), diff --git a/pkg/controllers/interruption/messages/statechange/model.go b/pkg/controllers/interruption/messages/statechange/model.go index b9c1d5b84f7a..144a1e7c4f0d 100644 --- a/pkg/controllers/interruption/messages/statechange/model.go +++ b/pkg/controllers/interruption/messages/statechange/model.go @@ -15,6 +15,8 @@ limitations under the License. package statechange import ( + "github.com/samber/lo" + "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/messages" ) @@ -35,6 +37,9 @@ func (m Message) EC2InstanceIDs() []string { return []string{m.Detail.InstanceID} } -func (Message) Kind() messages.Kind { - return messages.StateChangeKind +func (m Message) Kind() messages.Kind { + if lo.Contains([]string{"stopping", "stopped"}, m.Detail.State) { + return messages.InstanceStoppedKind + } + return messages.InstanceTerminatedKind } diff --git a/pkg/controllers/interruption/messages/types.go b/pkg/controllers/interruption/messages/types.go index 45f2f49def1e..99a6602cdf0a 100644 --- a/pkg/controllers/interruption/messages/types.go +++ b/pkg/controllers/interruption/messages/types.go @@ -35,11 +35,12 @@ type Message interface { type Kind string const ( - RebalanceRecommendationKind Kind = "RebalanceRecommendationKind" - ScheduledChangeKind Kind = "ScheduledChangeKind" - SpotInterruptionKind Kind = "SpotInterruptionKind" - StateChangeKind Kind = "StateChangeKind" - NoOpKind Kind = "NoOpKind" + RebalanceRecommendationKind Kind = "rebalance_recommendation" + ScheduledChangeKind Kind = "scheduled_change" + SpotInterruptionKind Kind = "spot_interrupted" + InstanceStoppedKind Kind = "instance_stopped" + InstanceTerminatedKind Kind = "instance_terminated" + NoOpKind Kind = "no_op" ) type Metadata struct { diff --git a/pkg/controllers/interruption/metrics.go b/pkg/controllers/interruption/metrics.go index 7fd01ca241d2..7b3f92468751 100644 --- a/pkg/controllers/interruption/metrics.go +++ b/pkg/controllers/interruption/metrics.go @@ -22,10 +22,8 @@ import ( ) const ( - interruptionSubsystem = "interruption" - messageTypeLabel = "message_type" - actionTypeLabel = "action_type" - terminationReasonLabel = "interruption" + interruptionSubsystem = "interruption" + messageTypeLabel = "message_type" ) var ( @@ -33,7 +31,7 @@ var ( prometheus.CounterOpts{ Namespace: metrics.Namespace, Subsystem: interruptionSubsystem, - Name: "received_messages", + Name: "received_messages_total", Help: "Count of messages received from the SQS queue. Broken down by message type and whether the message was actionable.", }, []string{messageTypeLabel}, @@ -42,7 +40,7 @@ var ( prometheus.CounterOpts{ Namespace: metrics.Namespace, Subsystem: interruptionSubsystem, - Name: "deleted_messages", + Name: "deleted_messages_total", Help: "Count of messages deleted from the SQS queue.", }, ) @@ -50,22 +48,13 @@ var ( prometheus.HistogramOpts{ Namespace: metrics.Namespace, Subsystem: interruptionSubsystem, - Name: "message_latency_time_seconds", - Help: "Length of time between message creation in queue and an action taken on the message by the controller.", + Name: "message_queue_duration_seconds", + Help: "Amount of time an interruption message is on the queue before it is processed by karpenter.", Buckets: metrics.DurationBuckets(), }, ) - actionsPerformed = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: metrics.Namespace, - Subsystem: interruptionSubsystem, - Name: "actions_performed", - Help: "Number of notification actions performed. Labeled by action", - }, - []string{actionTypeLabel}, - ) ) func init() { - crmetrics.Registry.MustRegister(receivedMessages, deletedMessages, messageLatency, actionsPerformed) + crmetrics.Registry.MustRegister(receivedMessages, deletedMessages, messageLatency) } diff --git a/pkg/controllers/interruption/suite_test.go b/pkg/controllers/interruption/suite_test.go index 52c2b0972add..58100a267f83 100644 --- a/pkg/controllers/interruption/suite_test.go +++ b/pkg/controllers/interruption/suite_test.go @@ -21,23 +21,24 @@ import ( "testing" "time" + "sigs.k8s.io/karpenter/pkg/metrics" + + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" servicesqs "github.com/aws/aws-sdk-go/service/sqs" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" - _ "knative.dev/pkg/system/testing" "sigs.k8s.io/controller-runtime/pkg/client" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/events" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" @@ -53,8 +54,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) const ( @@ -78,7 +79,7 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) fakeClock = &clock.FakeClock{} unavailableOfferingsCache = awscache.NewUnavailableOfferings() sqsapi = &fake.SQSAPI{} @@ -101,26 +102,31 @@ var _ = AfterEach(func() { }) var _ = Describe("InterruptionHandling", func() { - var node *v1.Node - var nodeClaim *corev1beta1.NodeClaim + var node *corev1.Node + var nodeClaim *karpv1.NodeClaim BeforeEach(func() { - nodeClaim, node = coretest.NodeClaimAndNode(corev1beta1.NodeClaim{ + nodeClaim, node = coretest.NodeClaimAndNode(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - corev1beta1.NodePoolLabelKey: "default", + karpv1.NodePoolLabelKey: "default", }, }, - Status: corev1beta1.NodeClaimStatus{ + Status: karpv1.NodeClaimStatus{ ProviderID: fake.RandomProviderID(), }, }) + metrics.NodeClaimsDisruptedTotal.Reset() }) Context("Processing Messages", func() { It("should delete the NodeClaim when receiving a spot interruption warning", func() { ExpectMessagesCreated(spotInterruptionMessage(lo.Must(utils.ParseInstanceID(nodeClaim.Status.ProviderID)))) ExpectApplied(ctx, env.Client, nodeClaim, node) - ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, controller) + ExpectMetricCounterValue(metrics.NodeClaimsDisruptedTotal, 1, map[string]string{ + metrics.ReasonLabel: "spot_interrupted", + "nodepool": "default", + }) Expect(sqsapi.ReceiveMessageBehavior.SuccessfulCalls()).To(Equal(1)) ExpectNotFound(ctx, env.Client, nodeClaim) Expect(sqsapi.DeleteMessageBehavior.SuccessfulCalls()).To(Equal(1)) @@ -129,23 +135,27 @@ var _ = Describe("InterruptionHandling", func() { ExpectMessagesCreated(scheduledChangeMessage(lo.Must(utils.ParseInstanceID(nodeClaim.Status.ProviderID)))) ExpectApplied(ctx, env.Client, nodeClaim, node) - ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, controller) + ExpectMetricCounterValue(metrics.NodeClaimsDisruptedTotal, 1, map[string]string{ + metrics.ReasonLabel: "scheduled_change", + "nodepool": "default", + }) Expect(sqsapi.ReceiveMessageBehavior.SuccessfulCalls()).To(Equal(1)) ExpectNotFound(ctx, env.Client, nodeClaim) Expect(sqsapi.DeleteMessageBehavior.SuccessfulCalls()).To(Equal(1)) }) It("should delete the NodeClaim when receiving a state change message", func() { - var nodeClaims []*corev1beta1.NodeClaim + var nodeClaims []*karpv1.NodeClaim var messages []interface{} for _, state := range []string{"terminated", "stopped", "stopping", "shutting-down"} { instanceID := fake.InstanceID() - nc, n := coretest.NodeClaimAndNode(corev1beta1.NodeClaim{ + nc, n := coretest.NodeClaimAndNode(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - corev1beta1.NodePoolLabelKey: "default", + karpv1.NodePoolLabelKey: "default", }, }, - Status: corev1beta1.NodeClaimStatus{ + Status: karpv1.NodeClaimStatus{ ProviderID: fake.ProviderID(instanceID), }, }) @@ -154,23 +164,31 @@ var _ = Describe("InterruptionHandling", func() { messages = append(messages, stateChangeMessage(instanceID, state)) } ExpectMessagesCreated(messages...) - ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, controller) + ExpectMetricCounterValue(metrics.NodeClaimsDisruptedTotal, 2, map[string]string{ + metrics.ReasonLabel: "instance_terminated", + "nodepool": "default", + }) + ExpectMetricCounterValue(metrics.NodeClaimsDisruptedTotal, 2, map[string]string{ + metrics.ReasonLabel: "instance_stopped", + "nodepool": "default", + }) Expect(sqsapi.ReceiveMessageBehavior.SuccessfulCalls()).To(Equal(1)) - ExpectNotFound(ctx, env.Client, lo.Map(nodeClaims, func(nc *corev1beta1.NodeClaim, _ int) client.Object { return nc })...) + ExpectNotFound(ctx, env.Client, lo.Map(nodeClaims, func(nc *karpv1.NodeClaim, _ int) client.Object { return nc })...) Expect(sqsapi.DeleteMessageBehavior.SuccessfulCalls()).To(Equal(4)) }) It("should handle multiple messages that cause nodeClaim deletion", func() { - var nodeClaims []*corev1beta1.NodeClaim + var nodeClaims []*karpv1.NodeClaim var instanceIDs []string for i := 0; i < 100; i++ { instanceID := fake.InstanceID() - nc, n := coretest.NodeClaimAndNode(corev1beta1.NodeClaim{ + nc, n := coretest.NodeClaimAndNode(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - corev1beta1.NodePoolLabelKey: "default", + karpv1.NodePoolLabelKey: "default", }, }, - Status: corev1beta1.NodeClaimStatus{ + Status: karpv1.NodeClaimStatus{ ProviderID: fake.ProviderID(instanceID), }, }) @@ -184,9 +202,9 @@ var _ = Describe("InterruptionHandling", func() { messages = append(messages, spotInterruptionMessage(id)) } ExpectMessagesCreated(messages...) - ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, controller) Expect(sqsapi.ReceiveMessageBehavior.SuccessfulCalls()).To(Equal(1)) - ExpectNotFound(ctx, env.Client, lo.Map(nodeClaims, func(nc *corev1beta1.NodeClaim, _ int) client.Object { return nc })...) + ExpectNotFound(ctx, env.Client, lo.Map(nodeClaims, func(nc *karpv1.NodeClaim, _ int) client.Object { return nc })...) Expect(sqsapi.DeleteMessageBehavior.SuccessfulCalls()).To(Equal(100)) }) It("should delete a message when the message can't be parsed", func() { @@ -200,7 +218,7 @@ var _ = Describe("InterruptionHandling", func() { ExpectMessagesCreated(badMessage) - ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, controller) Expect(sqsapi.ReceiveMessageBehavior.SuccessfulCalls()).To(Equal(1)) Expect(sqsapi.DeleteMessageBehavior.SuccessfulCalls()).To(Equal(1)) }) @@ -208,27 +226,27 @@ var _ = Describe("InterruptionHandling", func() { ExpectMessagesCreated(stateChangeMessage(lo.Must(utils.ParseInstanceID(nodeClaim.Status.ProviderID)), "creating")) ExpectApplied(ctx, env.Client, nodeClaim, node) - ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, controller) Expect(sqsapi.ReceiveMessageBehavior.SuccessfulCalls()).To(Equal(1)) ExpectExists(ctx, env.Client, nodeClaim) Expect(sqsapi.DeleteMessageBehavior.SuccessfulCalls()).To(Equal(1)) }) It("should mark the ICE cache for the offering when getting a spot interruption warning", func() { nodeClaim.Labels = lo.Assign(nodeClaim.Labels, map[string]string{ - v1.LabelTopologyZone: "coretest-zone-1a", - v1.LabelInstanceTypeStable: "t3.large", - corev1beta1.CapacityTypeLabelKey: corev1beta1.CapacityTypeSpot, + corev1.LabelTopologyZone: "coretest-zone-1a", + corev1.LabelInstanceTypeStable: "t3.large", + karpv1.CapacityTypeLabelKey: karpv1.CapacityTypeSpot, }) ExpectMessagesCreated(spotInterruptionMessage(lo.Must(utils.ParseInstanceID(nodeClaim.Status.ProviderID)))) ExpectApplied(ctx, env.Client, nodeClaim, node) - ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, controller) Expect(sqsapi.ReceiveMessageBehavior.SuccessfulCalls()).To(Equal(1)) ExpectNotFound(ctx, env.Client, nodeClaim) Expect(sqsapi.DeleteMessageBehavior.SuccessfulCalls()).To(Equal(1)) // Expect a t3.large in coretest-zone-1a to be added to the ICE cache - Expect(unavailableOfferingsCache.IsUnavailable("t3.large", "coretest-zone-1a", corev1beta1.CapacityTypeSpot)).To(BeTrue()) + Expect(unavailableOfferingsCache.IsUnavailable("t3.large", "coretest-zone-1a", karpv1.CapacityTypeSpot)).To(BeTrue()) }) }) }) @@ -236,15 +254,15 @@ var _ = Describe("InterruptionHandling", func() { var _ = Describe("Error Handling", func() { It("should send an error on polling when QueueNotExists", func() { sqsapi.ReceiveMessageBehavior.Error.Set(awsErrWithCode(servicesqs.ErrCodeQueueDoesNotExist), fake.MaxCalls(0)) - ExpectReconcileFailed(ctx, controller, types.NamespacedName{}) + _ = ExpectSingletonReconcileFailed(ctx, controller) }) It("should send an error on polling when AccessDenied", func() { sqsapi.ReceiveMessageBehavior.Error.Set(awsErrWithCode("AccessDenied"), fake.MaxCalls(0)) - ExpectReconcileFailed(ctx, controller, types.NamespacedName{}) + _ = ExpectSingletonReconcileFailed(ctx, controller) }) It("should not return an error when deleting a nodeClaim that is already deleted", func() { ExpectMessagesCreated(spotInterruptionMessage(fake.InstanceID())) - ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, controller) }) }) diff --git a/pkg/controllers/nodeclaim/garbagecollection/controller.go b/pkg/controllers/nodeclaim/garbagecollection/controller.go index dae66f7f6f1e..9387e9d0a331 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/controller.go +++ b/pkg/controllers/nodeclaim/garbagecollection/controller.go @@ -19,19 +19,22 @@ import ( "fmt" "time" + "github.com/awslabs/operatorpkg/singleton" "github.com/samber/lo" "go.uber.org/multierr" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/workqueue" - "knative.dev/pkg/logging" + "k8s.io/klog/v2" + controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/karpenter/pkg/cloudprovider" + "sigs.k8s.io/karpenter/pkg/operator/injection" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "sigs.k8s.io/karpenter/pkg/operator/controller" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" ) type Controller struct { @@ -48,11 +51,9 @@ func NewController(kubeClient client.Client, cloudProvider cloudprovider.CloudPr } } -func (c *Controller) Name() string { - return "nodeclaim.garbagecollection" -} +func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) { + ctx = injection.WithControllerName(ctx, "nodeclaim.garbagecollection") -func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { // We LIST machines on the CloudProvider BEFORE we grab Machines/Nodes on the cluster so that we make sure that, if // LISTing instances takes a long time, our information is more updated by the time we get to Machine and Node LIST // This works since our CloudProvider instances are deleted based on whether the Machine exists or not, not vise-versa @@ -60,18 +61,18 @@ func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconc if err != nil { return reconcile.Result{}, fmt.Errorf("listing cloudprovider machines, %w", err) } - managedRetrieved := lo.Filter(retrieved, func(nc *v1beta1.NodeClaim, _ int) bool { - return nc.Annotations[v1beta1.ManagedByAnnotationKey] != "" && nc.DeletionTimestamp.IsZero() + managedRetrieved := lo.Filter(retrieved, func(nc *karpv1.NodeClaim, _ int) bool { + return nc.DeletionTimestamp.IsZero() }) - nodeClaimList := &v1beta1.NodeClaimList{} + nodeClaimList := &karpv1.NodeClaimList{} if err = c.kubeClient.List(ctx, nodeClaimList); err != nil { return reconcile.Result{}, err } - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} if err = c.kubeClient.List(ctx, nodeList); err != nil { return reconcile.Result{}, err } - resolvedProviderIDs := sets.New[string](lo.FilterMap(nodeClaimList.Items, func(n v1beta1.NodeClaim, _ int) (string, bool) { + resolvedProviderIDs := sets.New[string](lo.FilterMap(nodeClaimList.Items, func(n karpv1.NodeClaim, _ int) (string, bool) { return n.Status.ProviderID, n.Status.ProviderID != "" })...) errs := make([]error, len(retrieved)) @@ -88,25 +89,28 @@ func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconc return reconcile.Result{RequeueAfter: lo.Ternary(c.successfulCount <= 20, time.Second*10, time.Minute*2)}, nil } -func (c *Controller) garbageCollect(ctx context.Context, nodeClaim *v1beta1.NodeClaim, nodeList *v1.NodeList) error { - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("provider-id", nodeClaim.Status.ProviderID)) +func (c *Controller) garbageCollect(ctx context.Context, nodeClaim *karpv1.NodeClaim, nodeList *corev1.NodeList) error { + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("provider-id", nodeClaim.Status.ProviderID)) if err := c.cloudProvider.Delete(ctx, nodeClaim); err != nil { return cloudprovider.IgnoreNodeClaimNotFoundError(err) } - logging.FromContext(ctx).Debugf("garbage collected cloudprovider instance") + log.FromContext(ctx).V(1).Info("garbage collected cloudprovider instance") // Go ahead and cleanup the node if we know that it exists to make scheduling go quicker - if node, ok := lo.Find(nodeList.Items, func(n v1.Node) bool { + if node, ok := lo.Find(nodeList.Items, func(n corev1.Node) bool { return n.Spec.ProviderID == nodeClaim.Status.ProviderID }); ok { if err := c.kubeClient.Delete(ctx, &node); err != nil { return client.IgnoreNotFound(err) } - logging.FromContext(ctx).With("node", node.Name).Debugf("garbage collected node") + log.FromContext(ctx).WithValues("Node", klog.KRef("", node.Name)).V(1).Info("garbage collected node") } return nil } -func (c *Controller) Builder(_ context.Context, m manager.Manager) controller.Builder { - return controller.NewSingletonManagedBy(m) +func (c *Controller) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("nodeclaim.garbagecollection"). + WatchesRawSource(singleton.Source()). + Complete(singleton.AsReconciler(c)) } diff --git a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go index 6527c20a2a4a..d72ff22e0081 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go @@ -21,22 +21,21 @@ import ( "testing" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/awslabs/operatorpkg/object" + "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/record" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + karpcloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/events" - "sigs.k8s.io/karpenter/pkg/operator/controller" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclaim/garbagecollection" "github.com/aws/karpenter-provider-aws/pkg/fake" @@ -45,14 +44,14 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) var ctx context.Context var awsEnv *test.Environment var env *coretest.Environment -var garbageCollectionController controller.Controller +var garbageCollectionController *garbagecollection.Controller var cloudProvider *cloudprovider.CloudProvider func TestAPIs(t *testing.T) { @@ -63,10 +62,10 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { ctx = options.ToContext(ctx, test.Options()) - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) awsEnv = test.NewEnvironment(ctx, env) cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), - env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider, awsEnv.SubnetProvider) + env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider) garbageCollectionController = garbagecollection.NewController(env.Client, cloudProvider) }) @@ -80,19 +79,21 @@ var _ = BeforeEach(func() { var _ = Describe("GarbageCollection", func() { var instance *ec2.Instance - var nodeClass *v1beta1.EC2NodeClass + var nodeClass *v1.EC2NodeClass var providerID string BeforeEach(func() { instanceID := fake.InstanceID() providerID = fake.ProviderID(instanceID) nodeClass = test.EC2NodeClass() - nodePool := coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nodePool := coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }, @@ -108,15 +109,15 @@ var _ = Describe("GarbageCollection", func() { Value: aws.String("owned"), }, { - Key: aws.String(corev1beta1.NodePoolLabelKey), + Key: aws.String(karpv1.NodePoolLabelKey), Value: aws.String(nodePool.Name), }, { - Key: aws.String(v1beta1.LabelNodeClass), + Key: aws.String(v1.LabelNodeClass), Value: aws.String(nodeClass.Name), }, { - Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Key: aws.String(v1.EKSClusterNameTagKey), Value: aws.String(options.FromContext(ctx).ClusterName), }, }, @@ -137,10 +138,10 @@ var _ = Describe("GarbageCollection", func() { instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) _, err := cloudProvider.Get(ctx, providerID) Expect(err).To(HaveOccurred()) - Expect(corecloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) + Expect(karpcloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) }) It("should delete an instance along with the node if there is no NodeClaim owner (to quicken scheduling)", func() { // Launch time was 1m ago @@ -152,10 +153,10 @@ var _ = Describe("GarbageCollection", func() { }) ExpectApplied(ctx, env.Client, node) - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) _, err := cloudProvider.Get(ctx, providerID) Expect(err).To(HaveOccurred()) - Expect(corecloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) + Expect(karpcloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) ExpectNotFound(ctx, env.Client, node) }) @@ -176,15 +177,15 @@ var _ = Describe("GarbageCollection", func() { Value: aws.String("owned"), }, { - Key: aws.String(corev1beta1.NodePoolLabelKey), + Key: aws.String(karpv1.NodePoolLabelKey), Value: aws.String("default"), }, { - Key: aws.String(v1beta1.LabelNodeClass), + Key: aws.String(v1.LabelNodeClass), Value: aws.String("default"), }, { - Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Key: aws.String(v1.EKSClusterNameTagKey), Value: aws.String(options.FromContext(ctx).ClusterName), }, }, @@ -200,7 +201,7 @@ var _ = Describe("GarbageCollection", func() { ) ids = append(ids, instanceID) } - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) wg := sync.WaitGroup{} for _, id := range ids { @@ -211,7 +212,7 @@ var _ = Describe("GarbageCollection", func() { _, err := cloudProvider.Get(ctx, fake.ProviderID(id)) Expect(err).To(HaveOccurred()) - Expect(corecloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) + Expect(karpcloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) }(id) } wg.Wait() @@ -219,7 +220,7 @@ var _ = Describe("GarbageCollection", func() { It("should not delete all instances if they all have NodeClaim owners", func() { // Generate 100 instances that have different instanceIDs var ids []string - var nodeClaims []*corev1beta1.NodeClaim + var nodeClaims []*karpv1.NodeClaim for i := 0; i < 100; i++ { instanceID := fake.InstanceID() awsEnv.EC2API.Instances.Store( @@ -244,13 +245,15 @@ var _ = Describe("GarbageCollection", func() { InstanceType: aws.String("m5.large"), }, ) - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, - Status: corev1beta1.NodeClaimStatus{ + Status: karpv1.NodeClaimStatus{ ProviderID: fake.ProviderID(instanceID), }, }) @@ -258,7 +261,7 @@ var _ = Describe("GarbageCollection", func() { nodeClaims = append(nodeClaims, nodeClaim) ids = append(ids, instanceID) } - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) wg := sync.WaitGroup{} for _, id := range ids { @@ -282,21 +285,21 @@ var _ = Describe("GarbageCollection", func() { instance.LaunchTime = aws.Time(time.Now()) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) _, err := cloudProvider.Get(ctx, providerID) Expect(err).NotTo(HaveOccurred()) }) It("should not delete an instance if it was not launched by a NodeClaim", func() { - // Remove the "karpenter.sh/managed-by" tag (this isn't launched by a machine) + // Remove the "karpenter.sh/nodepool" tag (this isn't launched by a machine) instance.Tags = lo.Reject(instance.Tags, func(t *ec2.Tag, _ int) bool { - return aws.StringValue(t.Key) == corev1beta1.ManagedByAnnotationKey + return aws.StringValue(t.Key) == karpv1.NodePoolLabelKey }) // Launch time was 1m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) _, err := cloudProvider.Get(ctx, providerID) Expect(err).NotTo(HaveOccurred()) }) @@ -305,13 +308,15 @@ var _ = Describe("GarbageCollection", func() { instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, - Status: corev1beta1.NodeClaimStatus{ + Status: karpv1.NodeClaimStatus{ ProviderID: providerID, }, }) @@ -320,7 +325,7 @@ var _ = Describe("GarbageCollection", func() { }) ExpectApplied(ctx, env.Client, nodeClaim, node) - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) _, err := cloudProvider.Get(ctx, providerID) Expect(err).ToNot(HaveOccurred()) ExpectExists(ctx, env.Client, node) @@ -328,7 +333,7 @@ var _ = Describe("GarbageCollection", func() { It("should not delete many instances or nodes if they already have NodeClaim owners that match it", func() { // Generate 100 instances that have different instanceIDs that have NodeClaims var ids []string - var nodes []*v1.Node + var nodes []*corev1.Node for i := 0; i < 100; i++ { instanceID := fake.InstanceID() awsEnv.EC2API.Instances.Store( @@ -343,11 +348,11 @@ var _ = Describe("GarbageCollection", func() { Value: aws.String("owned"), }, { - Key: aws.String(corev1beta1.NodePoolLabelKey), + Key: aws.String(karpv1.NodePoolLabelKey), Value: aws.String("default"), }, { - Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Key: aws.String(v1.EKSClusterNameTagKey), Value: aws.String(options.FromContext(ctx).ClusterName), }, }, @@ -361,13 +366,15 @@ var _ = Describe("GarbageCollection", func() { InstanceType: aws.String("m5.large"), }, ) - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, - Status: corev1beta1.NodeClaimStatus{ + Status: karpv1.NodeClaimStatus{ ProviderID: fake.ProviderID(instanceID), }, }) @@ -378,12 +385,12 @@ var _ = Describe("GarbageCollection", func() { ids = append(ids, instanceID) nodes = append(nodes, node) } - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) wg := sync.WaitGroup{} for i := range ids { wg.Add(1) - go func(id string, node *v1.Node) { + go func(id string, node *corev1.Node) { defer GinkgoRecover() defer wg.Done() diff --git a/pkg/controllers/nodeclaim/tagging/controller.go b/pkg/controllers/nodeclaim/tagging/controller.go index f3b43bf9dc4f..2589168b1b11 100644 --- a/pkg/controllers/nodeclaim/tagging/controller.go +++ b/pkg/controllers/nodeclaim/tagging/controller.go @@ -20,22 +20,26 @@ import ( "time" "k8s.io/apimachinery/pkg/api/equality" - "knative.dev/pkg/logging" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/karpenter/pkg/operator/injection" "github.com/samber/lo" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + "github.com/awslabs/operatorpkg/reasonable" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/utils" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" ) type Controller struct { @@ -43,33 +47,34 @@ type Controller struct { instanceProvider instance.Provider } -func NewController(kubeClient client.Client, instanceProvider instance.Provider) corecontroller.Controller { - return corecontroller.Typed[*corev1beta1.NodeClaim](kubeClient, &Controller{ +func NewController(kubeClient client.Client, instanceProvider instance.Provider) *Controller { + return &Controller{ kubeClient: kubeClient, instanceProvider: instanceProvider, - }) + } } -func (c *Controller) Name() string { - return "nodeclaim.tagging" -} +func (c *Controller) Reconcile(ctx context.Context, nodeClaim *karpv1.NodeClaim) (reconcile.Result, error) { + ctx = injection.WithControllerName(ctx, "nodeclaim.tagging") -func (c *Controller) Reconcile(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (reconcile.Result, error) { stored := nodeClaim.DeepCopy() if !isTaggable(nodeClaim) { return reconcile.Result{}, nil } - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("provider-id", nodeClaim.Status.ProviderID)) + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("provider-id", nodeClaim.Status.ProviderID)) id, err := utils.ParseInstanceID(nodeClaim.Status.ProviderID) if err != nil { // We don't throw an error here since we don't want to retry until the ProviderID has been updated. - logging.FromContext(ctx).Errorf("failed to parse instance ID, %w", err) + log.FromContext(ctx).Error(err, "failed parsing instance id") return reconcile.Result{}, nil } if err = c.tagInstance(ctx, nodeClaim, id); err != nil { return reconcile.Result{}, cloudprovider.IgnoreNodeClaimNotFoundError(err) } - nodeClaim.Annotations = lo.Assign(nodeClaim.Annotations, map[string]string{v1beta1.AnnotationInstanceTagged: "true"}) + nodeClaim.Annotations = lo.Assign(nodeClaim.Annotations, map[string]string{ + v1.AnnotationInstanceTagged: "true", + v1.AnnotationClusterNameTaggedCompatability: "true", + }) if !equality.Semantic.DeepEqual(nodeClaim, stored) { if err := c.kubeClient.Patch(ctx, nodeClaim, client.MergeFrom(stored)); err != nil { return reconcile.Result{}, client.IgnoreNotFound(err) @@ -78,21 +83,25 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClaim *corev1beta1.NodeC return reconcile.Result{}, nil } -func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { - return corecontroller.Adapt( - controllerruntime. - NewControllerManagedBy(m). - For(&corev1beta1.NodeClaim{}). - WithEventFilter(predicate.NewPredicateFuncs(func(o client.Object) bool { - return isTaggable(o.(*corev1beta1.NodeClaim)) - })), - ) +func (c *Controller) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("nodeclaim.tagging"). + For(&karpv1.NodeClaim{}). + WithEventFilter(predicate.NewPredicateFuncs(func(o client.Object) bool { + return isTaggable(o.(*karpv1.NodeClaim)) + })). + // Ok with using the default MaxConcurrentReconciles of 1 to avoid throttling from CreateTag write API + WithOptions(controller.Options{ + RateLimiter: reasonable.RateLimiter(), + }). + Complete(reconcile.AsReconciler(m.GetClient(), c)) } -func (c *Controller) tagInstance(ctx context.Context, nc *corev1beta1.NodeClaim, id string) error { +func (c *Controller) tagInstance(ctx context.Context, nc *karpv1.NodeClaim, id string) error { tags := map[string]string{ - v1beta1.TagName: nc.Status.NodeName, - v1beta1.TagNodeClaim: nc.Name, + v1.TagName: nc.Status.NodeName, + v1.TagNodeClaim: nc.Name, + v1.EKSClusterNameTagKey: options.FromContext(ctx).ClusterName, } // Remove tags which have been already populated @@ -114,9 +123,11 @@ func (c *Controller) tagInstance(ctx context.Context, nc *corev1beta1.NodeClaim, return nil } -func isTaggable(nc *corev1beta1.NodeClaim) bool { +func isTaggable(nc *karpv1.NodeClaim) bool { // Instance has already been tagged - if val := nc.Annotations[v1beta1.AnnotationInstanceTagged]; val == "true" { + instanceTagged := nc.Annotations[v1.AnnotationInstanceTagged] + clusterNameTagged := nc.Annotations[v1.AnnotationClusterNameTaggedCompatability] + if instanceTagged == "true" && clusterNameTagged == "true" { return false } // Node name is not yet known diff --git a/pkg/controllers/nodeclaim/tagging/suite_test.go b/pkg/controllers/nodeclaim/tagging/suite_test.go index 6addad95b22b..a2b2dfa872e6 100644 --- a/pkg/controllers/nodeclaim/tagging/suite_test.go +++ b/pkg/controllers/nodeclaim/tagging/suite_test.go @@ -19,37 +19,35 @@ import ( "fmt" "testing" - "github.com/samber/lo" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "github.com/samber/lo" + corev1 "k8s.io/apimachinery/pkg/apis/meta/v1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclaim/tagging" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/test" - "sigs.k8s.io/karpenter/pkg/operator/controller" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) var ctx context.Context var awsEnv *test.Environment var env *coretest.Environment -var taggingController controller.Controller +var taggingController *tagging.Controller func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -58,7 +56,7 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) awsEnv = test.NewEnvironment(ctx, env) @@ -90,11 +88,11 @@ var _ = Describe("TaggingController", func() { Value: aws.String("owned"), }, { - Key: aws.String(corev1beta1.NodePoolLabelKey), + Key: aws.String(karpv1.NodePoolLabelKey), Value: aws.String("default"), }, { - Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Key: aws.String(v1.EKSClusterNameTagKey), Value: aws.String(options.FromContext(ctx).ClusterName), }, }, @@ -110,39 +108,39 @@ var _ = Describe("TaggingController", func() { }) It("shouldn't tag instances without a Node", func() { - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ - Status: corev1beta1.NodeClaimStatus{ + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + Status: karpv1.NodeClaimStatus{ ProviderID: fake.ProviderID(*ec2Instance.InstanceId), }, }) ExpectApplied(ctx, env.Client, nodeClaim) - ExpectReconcileSucceeded(ctx, taggingController, client.ObjectKeyFromObject(nodeClaim)) - Expect(nodeClaim.Annotations).To(Not(HaveKey(v1beta1.AnnotationInstanceTagged))) + ExpectObjectReconciled(ctx, env.Client, taggingController, nodeClaim) + Expect(nodeClaim.Annotations).To(Not(HaveKey(v1.AnnotationInstanceTagged))) Expect(lo.ContainsBy(ec2Instance.Tags, func(tag *ec2.Tag) bool { - return *tag.Key == v1beta1.TagName + return *tag.Key == v1.TagName })).To(BeFalse()) }) It("shouldn't tag nodeclaim with a malformed provderID", func() { - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ - Status: corev1beta1.NodeClaimStatus{ + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + Status: karpv1.NodeClaimStatus{ ProviderID: "Bad providerID", NodeName: "default", }, }) ExpectApplied(ctx, env.Client, nodeClaim) - ExpectReconcileSucceeded(ctx, taggingController, client.ObjectKeyFromObject(nodeClaim)) - Expect(nodeClaim.Annotations).To(Not(HaveKey(v1beta1.AnnotationInstanceTagged))) + ExpectObjectReconciled(ctx, env.Client, taggingController, nodeClaim) + Expect(nodeClaim.Annotations).To(Not(HaveKey(v1.AnnotationInstanceTagged))) Expect(lo.ContainsBy(ec2Instance.Tags, func(tag *ec2.Tag) bool { - return *tag.Key == v1beta1.TagName + return *tag.Key == v1.TagName })).To(BeFalse()) }) It("should gracefully handle missing NodeClaim", func() { - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ - Status: corev1beta1.NodeClaimStatus{ + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + Status: karpv1.NodeClaimStatus{ ProviderID: fake.ProviderID(*ec2Instance.InstanceId), NodeName: "default", }, @@ -150,12 +148,12 @@ var _ = Describe("TaggingController", func() { ExpectApplied(ctx, env.Client, nodeClaim) ExpectDeleted(ctx, env.Client, nodeClaim) - ExpectReconcileSucceeded(ctx, taggingController, client.ObjectKeyFromObject(nodeClaim)) + ExpectObjectReconciled(ctx, env.Client, taggingController, nodeClaim) }) It("should gracefully handle missing instance", func() { - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ - Status: corev1beta1.NodeClaimStatus{ + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + Status: karpv1.NodeClaimStatus{ ProviderID: fake.ProviderID(*ec2Instance.InstanceId), NodeName: "default", }, @@ -163,35 +161,35 @@ var _ = Describe("TaggingController", func() { ExpectApplied(ctx, env.Client, nodeClaim) awsEnv.EC2API.Instances.Delete(*ec2Instance.InstanceId) - ExpectReconcileSucceeded(ctx, taggingController, client.ObjectKeyFromObject(nodeClaim)) - Expect(nodeClaim.Annotations).To(Not(HaveKey(v1beta1.AnnotationInstanceTagged))) + ExpectObjectReconciled(ctx, env.Client, taggingController, nodeClaim) + Expect(nodeClaim.Annotations).To(Not(HaveKey(v1.AnnotationInstanceTagged))) }) It("shouldn't tag nodeclaim with deletion timestamp set", func() { - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ - Status: corev1beta1.NodeClaimStatus{ + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + Status: karpv1.NodeClaimStatus{ ProviderID: fake.ProviderID(*ec2Instance.InstanceId), NodeName: "default", }, - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: corev1.ObjectMeta{ Finalizers: []string{"testing/finalizer"}, }, }) ExpectApplied(ctx, env.Client, nodeClaim) Expect(env.Client.Delete(ctx, nodeClaim)).To(Succeed()) - ExpectReconcileSucceeded(ctx, taggingController, client.ObjectKeyFromObject(nodeClaim)) - Expect(nodeClaim.Annotations).To(Not(HaveKey(v1beta1.AnnotationInstanceTagged))) + ExpectObjectReconciled(ctx, env.Client, taggingController, nodeClaim) + Expect(nodeClaim.Annotations).To(Not(HaveKey(v1.AnnotationInstanceTagged))) Expect(lo.ContainsBy(ec2Instance.Tags, func(tag *ec2.Tag) bool { - return *tag.Key == v1beta1.TagName + return *tag.Key == v1.TagName })).To(BeFalse()) }) DescribeTable( "should tag taggable instances", func(customTags ...string) { - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ - Status: corev1beta1.NodeClaimStatus{ + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + Status: karpv1.NodeClaimStatus{ ProviderID: fake.ProviderID(*ec2Instance.InstanceId), NodeName: "default", }, @@ -206,13 +204,14 @@ var _ = Describe("TaggingController", func() { awsEnv.EC2API.Instances.Store(*ec2Instance.InstanceId, ec2Instance) ExpectApplied(ctx, env.Client, nodeClaim) - ExpectReconcileSucceeded(ctx, taggingController, client.ObjectKeyFromObject(nodeClaim)) + ExpectObjectReconciled(ctx, env.Client, taggingController, nodeClaim) nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) - Expect(nodeClaim.Annotations).To(HaveKey(v1beta1.AnnotationInstanceTagged)) + Expect(nodeClaim.Annotations).To(HaveKey(v1.AnnotationInstanceTagged)) expectedTags := map[string]string{ - v1beta1.TagName: nodeClaim.Status.NodeName, - v1beta1.TagNodeClaim: nodeClaim.Name, + v1.TagName: nodeClaim.Status.NodeName, + v1.TagNodeClaim: nodeClaim.Name, + v1.EKSClusterNameTagKey: options.FromContext(ctx).ClusterName, } instanceTags := instance.NewInstance(ec2Instance).Tags for tag, value := range expectedTags { @@ -222,9 +221,12 @@ var _ = Describe("TaggingController", func() { Expect(instanceTags).To(HaveKeyWithValue(tag, value)) } }, - Entry("with only karpenter.k8s.aws/nodeclaim tag", v1beta1.TagName), - Entry("with only Name tag", v1beta1.TagNodeClaim), - Entry("with both Name and karpenter.k8s.aws/nodeclaim tags"), - Entry("with nothing to tag", v1beta1.TagName, v1beta1.TagNodeClaim), + Entry("with the karpenter.sh/nodeclaim tag", v1.TagName, v1.EKSClusterNameTagKey), + Entry("with the eks:eks-cluster-name tag", v1.TagName, v1.TagNodeClaim), + Entry("with the Name tag", v1.TagNodeClaim, v1.EKSClusterNameTagKey), + Entry("with the karpenter.sh/nodeclaim and eks:eks-cluster-name tags", v1.TagName), + Entry("with the Name and eks:eks-cluster-name tags", v1.TagNodeClaim), + Entry("with the karpenter.sh/nodeclaim and Name tags", v1.EKSClusterNameTagKey), + Entry("with nothing to tag", v1.TagNodeClaim, v1.EKSClusterNameTagKey, v1.TagName), ) }) diff --git a/pkg/controllers/nodeclass/hash/controller.go b/pkg/controllers/nodeclass/hash/controller.go index 8cc70c28c7bb..b9f1a0522c09 100644 --- a/pkg/controllers/nodeclass/hash/controller.go +++ b/pkg/controllers/nodeclass/hash/controller.go @@ -17,6 +17,8 @@ package hash import ( "context" + "github.com/aws/karpenter-provider-aws/pkg/utils" + "github.com/samber/lo" "go.uber.org/multierr" "k8s.io/apimachinery/pkg/api/equality" @@ -25,36 +27,37 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/karpenter/pkg/operator/injection" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" + "github.com/awslabs/operatorpkg/reasonable" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" ) -var _ corecontroller.TypedController[*v1beta1.EC2NodeClass] = (*Controller)(nil) - type Controller struct { kubeClient client.Client } -func NewController(kubeClient client.Client) corecontroller.Controller { - return corecontroller.Typed[*v1beta1.EC2NodeClass](kubeClient, &Controller{ +func NewController(kubeClient client.Client) *Controller { + return &Controller{ kubeClient: kubeClient, - }) + } } -func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (reconcile.Result, error) { +func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconcile.Result, error) { + ctx = injection.WithControllerName(ctx, "nodeclass.hash") + stored := nodeClass.DeepCopy() - if nodeClass.Annotations[v1beta1.AnnotationEC2NodeClassHashVersion] != v1beta1.EC2NodeClassHashVersion { + if nodeClass.Annotations[v1.AnnotationEC2NodeClassHashVersion] != v1.EC2NodeClassHashVersion { if err := c.updateNodeClaimHash(ctx, nodeClass); err != nil { return reconcile.Result{}, err } } nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash(), - v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, + v1.AnnotationEC2NodeClassHash: nodeClass.Hash(), + v1.AnnotationEC2NodeClassHashVersion: v1.EC2NodeClassHashVersion, }) if !equality.Semantic.DeepEqual(stored, nodeClass) { @@ -66,23 +69,23 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeCl return reconcile.Result{}, nil } -func (c *Controller) Name() string { - return "nodeclass.hash" -} - -func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { - return corecontroller.Adapt(controllerruntime. - NewControllerManagedBy(m). - For(&v1beta1.EC2NodeClass{}). - WithOptions(controller.Options{MaxConcurrentReconciles: 10})) +func (c *Controller) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("nodeclass.hash"). + For(&v1.EC2NodeClass{}). + WithOptions(controller.Options{ + RateLimiter: reasonable.RateLimiter(), + MaxConcurrentReconciles: 10, + }). + Complete(reconcile.AsReconciler(m.GetClient(), c)) } // Updating `ec2nodeclass-hash-version` annotation inside the karpenter controller means a breaking change has been made to the hash calculation. // `ec2nodeclass-hash` annotation on the EC2NodeClass will be updated, due to the breaking change, making the `ec2nodeclass-hash` on the NodeClaim different from // EC2NodeClass. Since, we cannot rely on the `ec2nodeclass-hash` on the NodeClaims, due to the breaking change, we will need to re-calculate the hash and update the annotation. // For more information on the Drift Hash Versioning: https://github.com/kubernetes-sigs/karpenter/blob/main/designs/drift-hash-versioning.md -func (c *Controller) updateNodeClaimHash(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) error { - ncList := &corev1beta1.NodeClaimList{} +func (c *Controller) updateNodeClaimHash(ctx context.Context, nodeClass *v1.EC2NodeClass) error { + ncList := &karpv1.NodeClaimList{} if err := c.kubeClient.List(ctx, ncList, client.MatchingFields{"spec.nodeClassRef.name": nodeClass.Name}); err != nil { return err } @@ -92,16 +95,26 @@ func (c *Controller) updateNodeClaimHash(ctx context.Context, nodeClass *v1beta1 nc := ncList.Items[i] stored := nc.DeepCopy() - if nc.Annotations[v1beta1.AnnotationEC2NodeClassHashVersion] != v1beta1.EC2NodeClassHashVersion { + nodePool, err := utils.ResolveNodePoolFromNodeClaim(ctx, c.kubeClient, &nc) + if err != nil { + return err + } + kubeletHash, err := utils.GetHashKubelet(nodePool, nodeClass) + if err != nil { + return err + } + + if nc.Annotations[v1.AnnotationEC2NodeClassHashVersion] != v1.EC2NodeClassHashVersion { nc.Annotations = lo.Assign(nc.Annotations, map[string]string{ - v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, + v1.AnnotationEC2NodeClassHashVersion: v1.EC2NodeClassHashVersion, }) // Any NodeClaim that is already drifted will remain drifted if the karpenter.k8s.aws/nodepool-hash-version doesn't match // Since the hashing mechanism has changed we will not be able to determine if the drifted status of the NodeClaim has changed - if nc.StatusConditions().GetCondition(corev1beta1.Drifted) == nil { + if nc.StatusConditions().Get(karpv1.ConditionTypeDrifted) == nil { nc.Annotations = lo.Assign(nc.Annotations, map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash(), + v1.AnnotationEC2NodeClassHash: nodeClass.Hash(), + v1.AnnotationKubeletCompatibilityHash: kubeletHash, }) } diff --git a/pkg/controllers/nodeclass/hash/suite_test.go b/pkg/controllers/nodeclass/hash/suite_test.go index a1e6bf19ff7f..7fd9b68190c2 100644 --- a/pkg/controllers/nodeclass/hash/suite_test.go +++ b/pkg/controllers/nodeclass/hash/suite_test.go @@ -16,36 +16,40 @@ package hash_test import ( "context" + "encoding/json" "testing" - "github.com/imdario/mergo" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - _ "knative.dev/pkg/system/testing" - "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/samber/lo" + "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + + "github.com/aws/karpenter-provider-aws/pkg/utils" + + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" "github.com/aws/aws-sdk-go/aws" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" + "github.com/awslabs/operatorpkg/object" + "github.com/imdario/mergo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/hash" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/test" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) var ctx context.Context var env *coretest.Environment var awsEnv *test.Environment -var hashController corecontroller.Controller +var hashController *hash.Controller func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -54,7 +58,7 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...), coretest.WithFieldIndexers(test.EC2NodeClassFieldIndexer(ctx))) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...), coretest.WithFieldIndexers(test.EC2NodeClassFieldIndexer(ctx))) ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) awsEnv = test.NewEnvironment(ctx, env) @@ -76,204 +80,365 @@ var _ = AfterEach(func() { }) var _ = Describe("NodeClass Hash Controller", func() { - var nodeClass *v1beta1.EC2NodeClass + var nodeClass *v1.EC2NodeClass + var nodePool *karpv1.NodePool BeforeEach(func() { - nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ - SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ + nodeClass = test.EC2NodeClass(v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, - SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, - AMISelectorTerms: []v1beta1.AMISelectorTerm{ + AMIFamily: lo.ToPtr(v1.AMIFamilyCustom), + AMISelectorTerms: []v1.AMISelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, }, }) + nodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + }, + }, + }, + }) }) - DescribeTable("should update the drift hash when static field is updated", func(changes *v1beta1.EC2NodeClass) { + DescribeTable("should update the drift hash when static field is updated", func(changes *v1.EC2NodeClass) { ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, hashController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) expectedHash := nodeClass.Hash() - Expect(nodeClass.ObjectMeta.Annotations[v1beta1.AnnotationEC2NodeClassHash]).To(Equal(expectedHash)) + Expect(nodeClass.ObjectMeta.Annotations[v1.AnnotationEC2NodeClassHash]).To(Equal(expectedHash)) Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride)).To(Succeed()) ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, hashController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) expectedHashTwo := nodeClass.Hash() - Expect(nodeClass.Annotations[v1beta1.AnnotationEC2NodeClassHash]).To(Equal(expectedHashTwo)) + Expect(nodeClass.Annotations[v1.AnnotationEC2NodeClassHash]).To(Equal(expectedHashTwo)) Expect(expectedHash).ToNot(Equal(expectedHashTwo)) }, - Entry("AMIFamily Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: aws.String(v1beta1.AMIFamilyBottlerocket)}}), - Entry("UserData Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), - Entry("Tags Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), - Entry("BlockDeviceMappings Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-test-3")}}}}), - Entry("DetailedMonitoring Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), - Entry("MetadataOptions Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: aws.String("disabled")}}}), - Entry("Context Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: aws.String("context-2")}}), + Entry("UserData Drift", &v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), + Entry("Tags Drift", &v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + Entry("BlockDeviceMappings Drift", &v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{DeviceName: aws.String("map-device-test-3")}}}}), + Entry("DetailedMonitoring Drift", &v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), + Entry("MetadataOptions Drift", &v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPEndpoint: aws.String("disabled")}}}), + Entry("Context Drift", &v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Context: aws.String("context-2")}}), ) + It("should update nodeClaim annotation kubelet hash if nodePool was configured using v1beta1 NodePool", func() { + kubeletConfig := &v1beta1.KubeletConfiguration{ + ClusterDNS: []string{"test-cluster-dns"}, + MaxPods: lo.ToPtr(int32(9383)), + PodsPerCore: lo.ToPtr(int32(9334283)), + } + kubeletConfigString, _ := json.Marshal(kubeletConfig) + nodePool.Annotations = lo.Assign(nodePool.Annotations, map[string]string{ + karpv1.KubeletCompatibilityAnnotationKey: string(kubeletConfigString), + }) + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{karpv1.NodePoolLabelKey: nodePool.Name}, + Annotations: map[string]string{ + v1.AnnotationEC2NodeClassHash: "123456", + v1.AnnotationEC2NodeClassHashVersion: "test", + v1.AnnotationKubeletCompatibilityHash: "123456", + }, + }, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodeClass, nodeClaim, nodePool) + expectedHash, _ := utils.GetHashKubelet(nodePool, nodeClass) + + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + Expect(nodeClaim.Annotations[v1.AnnotationKubeletCompatibilityHash]).To(Equal(expectedHash)) + }) + It("should update nodeClaim annotation kubelet hash when kubelet is configured using ec2nodeClass", func() { + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{karpv1.NodePoolLabelKey: nodePool.Name}, + Annotations: map[string]string{ + v1.AnnotationEC2NodeClassHash: "123456", + v1.AnnotationEC2NodeClassHashVersion: "test", + v1.AnnotationKubeletCompatibilityHash: "123456", + }, + }, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + }, + }) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + ClusterDNS: []string{"test-cluster-dns"}, + MaxPods: lo.ToPtr(int32(9383)), + PodsPerCore: lo.ToPtr(int32(9334283)), + } + ExpectApplied(ctx, env.Client, nodeClass, nodeClaim, nodePool) + expectedHash, _ := utils.GetHashKubelet(nodePool, nodeClass) + + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + Expect(nodeClaim.Annotations[v1.AnnotationKubeletCompatibilityHash]).To(Equal(expectedHash)) + }) + It("should not update nodeClaim annotation kubelet hash if annotation is same as kubelet configuration on nodeClass", func() { + kubeletConfig := &v1beta1.KubeletConfiguration{ + ClusterDNS: []string{"test-cluster-dns"}, + MaxPods: lo.ToPtr(int32(9383)), + PodsPerCore: lo.ToPtr(int32(9334283)), + } + kubeletConfigString, _ := json.Marshal(kubeletConfig) + nodePool.Annotations = lo.Assign(nodePool.Annotations, map[string]string{ + karpv1.KubeletCompatibilityAnnotationKey: string(kubeletConfigString), + }) + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{karpv1.NodePoolLabelKey: nodePool.Name}, + Annotations: map[string]string{ + v1.AnnotationEC2NodeClassHash: "123456", + v1.AnnotationEC2NodeClassHashVersion: "test", + }, + }, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodeClass, nodeClaim, nodePool) + + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + hashBefore := nodeClaim.Annotations[v1.AnnotationKubeletCompatibilityHash] + + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + ClusterDNS: []string{"test-cluster-dns"}, + MaxPods: lo.ToPtr(int32(9383)), + PodsPerCore: lo.ToPtr(int32(9334283)), + } + nodePool.Annotations = nil + ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + Expect(nodeClaim.Annotations[v1.AnnotationKubeletCompatibilityHash]).To(Equal(hashBefore)) + }) It("should not update the drift hash when dynamic field is updated", func() { ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, hashController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) expectedHash := nodeClass.Hash() - Expect(nodeClass.Annotations[v1beta1.AnnotationEC2NodeClassHash]).To(Equal(expectedHash)) + Expect(nodeClass.Annotations[v1.AnnotationEC2NodeClassHash]).To(Equal(expectedHash)) - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { ID: "subnet-test1", }, } - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { ID: "sg-test1", }, } - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ { Tags: map[string]string{"ami-test-key": "ami-test-value"}, }, } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, hashController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Annotations[v1beta1.AnnotationEC2NodeClassHash]).To(Equal(expectedHash)) + Expect(nodeClass.Annotations[v1.AnnotationEC2NodeClassHash]).To(Equal(expectedHash)) }) It("should update ec2nodeclass-hash-version annotation when the ec2nodeclass-hash-version on the NodeClass does not match with the controller hash version", func() { nodeClass.Annotations = map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "abceduefed", - v1beta1.AnnotationEC2NodeClassHashVersion: "test", + v1.AnnotationEC2NodeClassHash: "abceduefed", + v1.AnnotationEC2NodeClassHashVersion: "test", } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, hashController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) expectedHash := nodeClass.Hash() // Expect ec2nodeclass-hash on the NodeClass to be updated - Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) - Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHash, expectedHash)) + Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHashVersion, v1.EC2NodeClassHashVersion)) }) It("should update ec2nodeclass-hash-versions on all NodeClaims when the ec2nodeclass-hash-version does not match with the controller hash version", func() { nodeClass.Annotations = map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "abceduefed", - v1beta1.AnnotationEC2NodeClassHashVersion: "test", + v1.AnnotationEC2NodeClassHash: "abceduefed", + v1.AnnotationEC2NodeClassHashVersion: "test", } - nodeClaimOne := coretest.NodeClaim(corev1beta1.NodeClaim{ + nodeClaimOne := coretest.NodeClaim(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{karpv1.NodePoolLabelKey: nodePool.Name}, Annotations: map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "123456", - v1beta1.AnnotationEC2NodeClassHashVersion: "test", + v1.AnnotationEC2NodeClassHash: "123456", + v1.AnnotationEC2NodeClassHashVersion: "test", }, }, - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) - nodeClaimTwo := coretest.NodeClaim(corev1beta1.NodeClaim{ + nodeClaimTwo := coretest.NodeClaim(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{karpv1.NodePoolLabelKey: nodePool.Name}, Annotations: map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "123456", - v1beta1.AnnotationEC2NodeClassHashVersion: "test", + v1.AnnotationEC2NodeClassHash: "123456", + v1.AnnotationEC2NodeClassHashVersion: "test", }, }, - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) - ExpectApplied(ctx, env.Client, nodeClass, nodeClaimOne, nodeClaimTwo) + ExpectApplied(ctx, env.Client, nodeClass, nodeClaimOne, nodeClaimTwo, nodePool) - ExpectReconcileSucceeded(ctx, hashController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) nodeClaimOne = ExpectExists(ctx, env.Client, nodeClaimOne) nodeClaimTwo = ExpectExists(ctx, env.Client, nodeClaimTwo) expectedHash := nodeClass.Hash() // Expect ec2nodeclass-hash on the NodeClaims to be updated - Expect(nodeClaimOne.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) - Expect(nodeClaimOne.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) - Expect(nodeClaimTwo.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) - Expect(nodeClaimTwo.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + Expect(nodeClaimOne.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHash, expectedHash)) + Expect(nodeClaimOne.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHashVersion, v1.EC2NodeClassHashVersion)) + Expect(nodeClaimTwo.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHash, expectedHash)) + Expect(nodeClaimTwo.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHashVersion, v1.EC2NodeClassHashVersion)) }) It("should not update ec2nodeclass-hash on all NodeClaims when the ec2nodeclass-hash-version matches the controller hash version", func() { nodeClass.Annotations = map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "abceduefed", - v1beta1.AnnotationEC2NodeClassHashVersion: "test-version", + v1.AnnotationEC2NodeClassHash: "abceduefed", + v1.AnnotationEC2NodeClassHashVersion: "test-version", } - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{karpv1.NodePoolLabelKey: nodePool.Name}, Annotations: map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "1234564654", - v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, + v1.AnnotationEC2NodeClassHash: "1234564654", + v1.AnnotationEC2NodeClassHashVersion: v1.EC2NodeClassHashVersion, }, }, - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) - ExpectApplied(ctx, env.Client, nodeClass, nodeClaim) + ExpectApplied(ctx, env.Client, nodeClass, nodeClaim, nodePool) - ExpectReconcileSucceeded(ctx, hashController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) expectedHash := nodeClass.Hash() // Expect ec2nodeclass-hash on the NodeClass to be updated - Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) - Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHash, expectedHash)) + Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHashVersion, v1.EC2NodeClassHashVersion)) // Expect ec2nodeclass-hash on the NodeClaims to stay the same - Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, "1234564654")) - Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHash, "1234564654")) + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHashVersion, v1.EC2NodeClassHashVersion)) }) It("should not update ec2nodeclass-hash on the NodeClaim if it's drifted and the ec2nodeclass-hash-version does not match the controller hash version", func() { nodeClass.Annotations = map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "abceduefed", - v1beta1.AnnotationEC2NodeClassHashVersion: "test", + v1.AnnotationEC2NodeClassHash: "abceduefed", + v1.AnnotationEC2NodeClassHashVersion: "test", } - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{karpv1.NodePoolLabelKey: nodePool.Name}, Annotations: map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "123456", - v1beta1.AnnotationEC2NodeClassHashVersion: "test", + v1.AnnotationEC2NodeClassHash: "123456", + v1.AnnotationEC2NodeClassHashVersion: "test", }, }, - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) - nodeClaim.StatusConditions().MarkTrue(corev1beta1.Drifted) - ExpectApplied(ctx, env.Client, nodeClass, nodeClaim) + nodeClaim.StatusConditions().SetTrue(karpv1.ConditionTypeDrifted) + ExpectApplied(ctx, env.Client, nodeClass, nodeClaim, nodePool) - ExpectReconcileSucceeded(ctx, hashController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) // Expect ec2nodeclass-hash on the NodeClaims to stay the same - Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, "123456")) - Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHash, "123456")) + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHashVersion, v1.EC2NodeClassHashVersion)) + }) + It("should update nodeClaim annotation kubelet hash when using a standalone nodeClaim", func() { + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.AnnotationEC2NodeClassHash: "123456", + v1.AnnotationEC2NodeClassHashVersion: "test", + }, + }, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + }, + }) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + ClusterDNS: []string{"test-cluster-dns"}, + MaxPods: lo.ToPtr(int32(9383)), + PodsPerCore: lo.ToPtr(int32(9334283)), + } + ExpectApplied(ctx, env.Client, nodeClass, nodeClaim, nodePool) + expectedHash, _ := utils.GetHashKubelet(nil, nodeClass) + + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + Expect(nodeClaim.Annotations[v1.AnnotationKubeletCompatibilityHash]).To(Equal(expectedHash)) }) }) diff --git a/pkg/controllers/nodeclass/status/ami.go b/pkg/controllers/nodeclass/status/ami.go index 71a80f51f130..3a23a1165808 100644 --- a/pkg/controllers/nodeclass/status/ami.go +++ b/pkg/controllers/nodeclass/status/ami.go @@ -21,9 +21,12 @@ import ( "time" "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" ) @@ -31,28 +34,37 @@ type AMI struct { amiProvider amifamily.Provider } -func (a *AMI) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (reconcile.Result, error) { - amis, err := a.amiProvider.Get(ctx, nodeClass, &amifamily.Options{}) +func (a *AMI) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconcile.Result, error) { + if nodeClass.UbuntuIncompatible() { + nodeClass.StatusConditions().SetFalse(v1.ConditionTypeAMIsReady, "AMINotFound", "Ubuntu AMI discovery is not supported at v1, refer to the upgrade guide (https://karpenter.sh/docs/upgrading/upgrade-guide/#upgrading-to-100)") + return reconcile.Result{}, nil + } + amis, err := a.amiProvider.List(ctx, nodeClass) if err != nil { - return reconcile.Result{}, err + return reconcile.Result{}, fmt.Errorf("getting amis, %w", err) } if len(amis) == 0 { nodeClass.Status.AMIs = nil - return reconcile.Result{}, fmt.Errorf("no amis exist given constraints") + nodeClass.StatusConditions().SetFalse(v1.ConditionTypeAMIsReady, "AMINotFound", "AMISelector did not match any AMIs") + return reconcile.Result{}, nil } - nodeClass.Status.AMIs = lo.Map(amis, func(ami amifamily.AMI, _ int) v1beta1.AMI { - reqs := ami.Requirements.NodeSelectorRequirements() + nodeClass.Status.AMIs = lo.Map(amis, func(ami amifamily.AMI, _ int) v1.AMI { + reqs := lo.Map(ami.Requirements.NodeSelectorRequirements(), func(item karpv1.NodeSelectorRequirementWithMinValues, _ int) corev1.NodeSelectorRequirement { + return item.NodeSelectorRequirement + }) + sort.Slice(reqs, func(i, j int) bool { if len(reqs[i].Key) != len(reqs[j].Key) { return len(reqs[i].Key) < len(reqs[j].Key) } return reqs[i].Key < reqs[j].Key }) - return v1beta1.AMI{ + return v1.AMI{ Name: ami.Name, ID: ami.AmiID, Requirements: reqs, } }) + nodeClass.StatusConditions().SetTrue(v1.ConditionTypeAMIsReady) return reconcile.Result{RequeueAfter: 5 * time.Minute}, nil } diff --git a/pkg/controllers/nodeclass/status/ami_test.go b/pkg/controllers/nodeclass/status/ami_test.go index 0e442cbf33a1..8ff07296c27f 100644 --- a/pkg/controllers/nodeclass/status/ami_test.go +++ b/pkg/controllers/nodeclass/status/ami_test.go @@ -18,16 +18,13 @@ import ( "fmt" "time" - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - _ "knative.dev/pkg/system/testing" - "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/test" . "github.com/onsi/ginkgo/v2" @@ -37,19 +34,20 @@ import ( var _ = Describe("NodeClass AMI Status Controller", func() { BeforeEach(func() { - nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ - SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ + nodeClass = test.EC2NodeClass(v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, - SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, - AMISelectorTerms: []v1beta1.AMISelectorTerm{ + AMIFamily: lo.ToPtr(v1.AMIFamilyCustom), + AMISelectorTerms: []v1.AMISelectorTerm{ { Tags: map[string]string{"*": "*"}, }, @@ -91,13 +89,25 @@ var _ = Describe("NodeClass AMI Status Controller", func() { }, }) }) + It("should fail to resolve AMIs if the nodeclass has ubuntu incompatible annotation", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2) + nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1.AnnotationUbuntuCompatibilityKey: v1.AnnotationUbuntuCompatibilityIncompatible}) + ExpectApplied(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + cond := nodeClass.StatusConditions().Get(v1.ConditionTypeAMIsReady) + Expect(cond.IsTrue()).To(BeFalse()) + Expect(cond.Message).To(Equal("Ubuntu AMI discovery is not supported at v1, refer to the upgrade guide (https://karpenter.sh/docs/upgrading/upgrade-guide/#upgrading-to-100)")) + Expect(cond.Reason).To(Equal("AMINotFound")) + + }) It("should resolve amiSelector AMIs and requirements into status", func() { version := lo.Must(awsEnv.VersionProvider.Get(ctx)) awsEnv.SSMAPI.Parameters = map[string]string{ - fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): "ami-id-123", - fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version): "ami-id-456", - fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, fmt.Sprintf("-%s", corev1beta1.ArchitectureArm64)): "ami-id-789", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): "ami-id-123", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version): "ami-id-456", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, fmt.Sprintf("-%s", karpv1.ArchitectureArm64)): "ami-id-789", } awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ @@ -126,7 +136,7 @@ var _ = Describe("NodeClass AMI Status Controller", func() { Name: aws.String("test-ami-3"), ImageId: aws.String("ami-id-789"), CreationDate: aws.String(time.Now().Add(2 * time.Minute).Format(time.RFC3339)), - Architecture: aws.String("x86_64"), + Architecture: aws.String("arm64"), Tags: []*ec2.Tag{ {Key: aws.String("Name"), Value: aws.String("test-ami-3")}, {Key: aws.String("foo"), Value: aws.String("bar")}, @@ -134,100 +144,83 @@ var _ = Describe("NodeClass AMI Status Controller", func() { }, }, }) - nodeClass.Spec.AMISelectorTerms = nil + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.AMIs).To(Equal([]v1beta1.AMI{ + Expect(len(nodeClass.Status.AMIs)).To(Equal(4)) + Expect(nodeClass.Status.AMIs).To(ContainElements([]v1.AMI{ { Name: "test-ami-3", ID: "ami-id-789", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []corev1.NodeSelectorRequirement{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureArm64}, - }, + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureArm64}, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpDoesNotExist, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpDoesNotExist, }, }, }, { Name: "test-ami-2", ID: "ami-id-456", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []corev1.NodeSelectorRequirement{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, - }, + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpExists, - }, + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpExists, }, }, }, { Name: "test-ami-2", ID: "ami-id-456", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []corev1.NodeSelectorRequirement{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, - }, + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpExists, - }, + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpExists, }, }, }, { Name: "test-ami-1", ID: "ami-id-123", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []corev1.NodeSelectorRequirement{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, - }, + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpDoesNotExist, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpDoesNotExist, }, }, }, })) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeAMIsReady)).To(BeTrue()) }) It("should resolve amiSelector AMis and requirements into status when all SSM aliases don't resolve", func() { version := lo.Must(awsEnv.VersionProvider.Get(ctx)) @@ -236,8 +229,9 @@ var _ = Describe("NodeClass AMI Status Controller", func() { fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id", version): "ami-id-123", fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id", version): "ami-id-456", } - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket - nodeClass.Spec.AMISelectorTerms = nil + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ + Alias: "bottlerocket@latest", + }} awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ Images: []*ec2.Image{ { @@ -263,84 +257,77 @@ var _ = Describe("NodeClass AMI Status Controller", func() { }, }) ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.AMIs).To(Equal([]v1beta1.AMI{ + Expect(len(nodeClass.Status.AMIs)).To(Equal(2)) + Expect(nodeClass.Status.AMIs).To(ContainElements([]v1.AMI{ { Name: "test-ami-2", ID: "ami-id-456", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []corev1.NodeSelectorRequirement{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureArm64}, - }, + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureArm64}, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpDoesNotExist, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpDoesNotExist, }, }, }, { Name: "test-ami-1", ID: "ami-id-123", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []corev1.NodeSelectorRequirement{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, - }, + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpDoesNotExist, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpDoesNotExist, }, }, }, })) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeAMIsReady)).To(BeTrue()) }) It("Should resolve a valid AMI selector", func() { ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) Expect(nodeClass.Status.AMIs).To(Equal( - []v1beta1.AMI{ + []v1.AMI{ { Name: "test-ami-3", ID: "ami-test3", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ - { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: "kubernetes.io/arch", - Operator: "In", - Values: []string{ - "amd64", - }, - }, - }, + Requirements: []corev1.NodeSelectorRequirement{{ + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, + }, }, }, }, )) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeAMIsReady)).To(BeTrue()) + }) + It("should get error when resolving AMIs and have status condition set to false", func() { + awsEnv.EC2API.NextError.Set(fmt.Errorf("unable to resolve AMI")) + ExpectApplied(ctx, env.Client, nodeClass) + _ = ExpectObjectReconcileFailed(ctx, env.Client, statusController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeAMIsReady)).To(BeFalse()) }) }) diff --git a/pkg/controllers/nodeclass/status/controller.go b/pkg/controllers/nodeclass/status/controller.go index e39981d9f510..1714da820348 100644 --- a/pkg/controllers/nodeclass/status/controller.go +++ b/pkg/controllers/nodeclass/status/controller.go @@ -25,11 +25,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/karpenter/pkg/operator/injection" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" "sigs.k8s.io/karpenter/pkg/utils/result" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + "github.com/awslabs/operatorpkg/reasonable" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" @@ -37,10 +39,8 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/providers/subnet" ) -var _ corecontroller.TypedController[*v1beta1.EC2NodeClass] = (*Controller)(nil) - type nodeClassStatusReconciler interface { - Reconcile(context.Context, *v1beta1.EC2NodeClass) (reconcile.Result, error) + Reconcile(context.Context, *v1.EC2NodeClass) (reconcile.Result, error) } type Controller struct { @@ -50,26 +50,28 @@ type Controller struct { instanceprofile *InstanceProfile subnet *Subnet securitygroup *SecurityGroup - launchtemplate *LaunchTemplate + readiness *Readiness //TODO : Remove this when we have sub status conditions } func NewController(kubeClient client.Client, subnetProvider subnet.Provider, securityGroupProvider securitygroup.Provider, - amiProvider amifamily.Provider, instanceProfileProvider instanceprofile.Provider, launchTemplateProvider launchtemplate.Provider) corecontroller.Controller { - return corecontroller.Typed[*v1beta1.EC2NodeClass](kubeClient, &Controller{ + amiProvider amifamily.Provider, instanceProfileProvider instanceprofile.Provider, launchTemplateProvider launchtemplate.Provider) *Controller { + return &Controller{ kubeClient: kubeClient, ami: &AMI{amiProvider: amiProvider}, subnet: &Subnet{subnetProvider: subnetProvider}, securitygroup: &SecurityGroup{securityGroupProvider: securityGroupProvider}, instanceprofile: &InstanceProfile{instanceProfileProvider: instanceProfileProvider}, - launchtemplate: &LaunchTemplate{launchTemplateProvider: launchTemplateProvider}, - }) + readiness: &Readiness{launchTemplateProvider: launchTemplateProvider}, + } } -func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (reconcile.Result, error) { - if !controllerutil.ContainsFinalizer(nodeClass, v1beta1.TerminationFinalizer) { +func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconcile.Result, error) { + ctx = injection.WithControllerName(ctx, "nodeclass.status") + + if !controllerutil.ContainsFinalizer(nodeClass, v1.TerminationFinalizer) { stored := nodeClass.DeepCopy() - controllerutil.AddFinalizer(nodeClass, v1beta1.TerminationFinalizer) + controllerutil.AddFinalizer(nodeClass, v1.TerminationFinalizer) if err := c.kubeClient.Patch(ctx, nodeClass, client.MergeFrom(stored)); err != nil { return reconcile.Result{}, err } @@ -83,7 +85,7 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeCl c.subnet, c.securitygroup, c.instanceprofile, - c.launchtemplate, + c.readiness, } { res, err := reconciler.Reconcile(ctx, nodeClass) errs = multierr.Append(errs, err) @@ -101,13 +103,13 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeCl return result.Min(results...), nil } -func (c *Controller) Name() string { - return "nodeclass.status" -} - -func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { - return corecontroller.Adapt(controllerruntime. - NewControllerManagedBy(m). - For(&v1beta1.EC2NodeClass{}). - WithOptions(controller.Options{MaxConcurrentReconciles: 10})) +func (c *Controller) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("nodeclass.status"). + For(&v1.EC2NodeClass{}). + WithOptions(controller.Options{ + RateLimiter: reasonable.RateLimiter(), + MaxConcurrentReconciles: 10, + }). + Complete(reconcile.AsReconciler(m.GetClient(), c)) } diff --git a/pkg/controllers/nodeclass/status/instanceprofile.go b/pkg/controllers/nodeclass/status/instanceprofile.go index 10988803db5c..9ff92d299bae 100644 --- a/pkg/controllers/nodeclass/status/instanceprofile.go +++ b/pkg/controllers/nodeclass/status/instanceprofile.go @@ -21,7 +21,7 @@ import ( "github.com/samber/lo" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" ) @@ -29,7 +29,7 @@ type InstanceProfile struct { instanceProfileProvider instanceprofile.Provider } -func (ip *InstanceProfile) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (reconcile.Result, error) { +func (ip *InstanceProfile) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconcile.Result, error) { if nodeClass.Spec.Role != "" { name, err := ip.instanceProfileProvider.Create(ctx, nodeClass) if err != nil { @@ -39,6 +39,6 @@ func (ip *InstanceProfile) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2 } else { nodeClass.Status.InstanceProfile = lo.FromPtr(nodeClass.Spec.InstanceProfile) } - + nodeClass.StatusConditions().SetTrue(v1.ConditionTypeInstanceProfileReady) return reconcile.Result{}, nil } diff --git a/pkg/controllers/nodeclass/status/instanceprofile_test.go b/pkg/controllers/nodeclass/status/instanceprofile_test.go index 386fd6dddc1b..75f3fa7df1ad 100644 --- a/pkg/controllers/nodeclass/status/instanceprofile_test.go +++ b/pkg/controllers/nodeclass/status/instanceprofile_test.go @@ -15,12 +15,13 @@ limitations under the License. package status_test import ( - "github.com/samber/lo" - _ "knative.dev/pkg/system/testing" - "sigs.k8s.io/controller-runtime/pkg/client" + "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" + "github.com/samber/lo" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" @@ -38,14 +39,20 @@ var _ = Describe("NodeClass InstanceProfile Status Controller", func() { It("should create the instance profile when it doesn't exist", func() { nodeClass.Spec.Role = "test-role" ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(1)) Expect(awsEnv.IAMAPI.InstanceProfiles[profileName].Roles).To(HaveLen(1)) Expect(*awsEnv.IAMAPI.InstanceProfiles[profileName].Roles[0].RoleName).To(Equal("test-role")) + Expect(awsEnv.IAMAPI.InstanceProfiles[profileName].Tags).To(ContainElements( + &iam.Tag{Key: lo.ToPtr(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: lo.ToPtr("owned")}, + &iam.Tag{Key: lo.ToPtr(v1.LabelNodeClass), Value: lo.ToPtr(nodeClass.Name)}, + &iam.Tag{Key: lo.ToPtr(v1.EKSClusterNameTagKey), Value: lo.ToPtr(options.FromContext(ctx).ClusterName)}, + )) nodeClass = ExpectExists(ctx, env.Client, nodeClass) Expect(nodeClass.Status.InstanceProfile).To(Equal(profileName)) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeInstanceProfileReady)).To(BeTrue()) }) It("should add the role to the instance profile when it exists without a role", func() { awsEnv.IAMAPI.InstanceProfiles = map[string]*iam.InstanceProfile{ @@ -57,7 +64,7 @@ var _ = Describe("NodeClass InstanceProfile Status Controller", func() { nodeClass.Spec.Role = "test-role" ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(1)) Expect(awsEnv.IAMAPI.InstanceProfiles[profileName].Roles).To(HaveLen(1)) @@ -65,6 +72,7 @@ var _ = Describe("NodeClass InstanceProfile Status Controller", func() { nodeClass = ExpectExists(ctx, env.Client, nodeClass) Expect(nodeClass.Status.InstanceProfile).To(Equal(profileName)) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeInstanceProfileReady)).To(BeTrue()) }) It("should update the role for the instance profile when the wrong role exists", func() { awsEnv.IAMAPI.InstanceProfiles = map[string]*iam.InstanceProfile{ @@ -81,7 +89,7 @@ var _ = Describe("NodeClass InstanceProfile Status Controller", func() { nodeClass.Spec.Role = "test-role" ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(1)) Expect(awsEnv.IAMAPI.InstanceProfiles[profileName].Roles).To(HaveLen(1)) @@ -89,6 +97,40 @@ var _ = Describe("NodeClass InstanceProfile Status Controller", func() { nodeClass = ExpectExists(ctx, env.Client, nodeClass) Expect(nodeClass.Status.InstanceProfile).To(Equal(profileName)) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeInstanceProfileReady)).To(BeTrue()) + }) + It("should add the eks:eks-cluster-name tag when the tag doesn't exist", func() { + awsEnv.IAMAPI.InstanceProfiles = map[string]*iam.InstanceProfile{ + profileName: { + InstanceProfileId: aws.String(fake.InstanceProfileID()), + InstanceProfileName: aws.String(profileName), + Roles: []*iam.Role{ + { + RoleName: aws.String("other-role"), + }, + }, + Tags: []*iam.Tag{ + { + Key: lo.ToPtr(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), + Value: lo.ToPtr("owned"), + }, + { + Key: lo.ToPtr(v1.LabelNodeClass), + Value: lo.ToPtr(nodeClass.Name), + }, + }, + }, + } + + ExpectApplied(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) + + Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(1)) + Expect(awsEnv.IAMAPI.InstanceProfiles[profileName].Tags).To(ContainElements( + &iam.Tag{Key: lo.ToPtr(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: lo.ToPtr("owned")}, + &iam.Tag{Key: lo.ToPtr(v1.LabelNodeClass), Value: lo.ToPtr(nodeClass.Name)}, + &iam.Tag{Key: lo.ToPtr(v1.EKSClusterNameTagKey), Value: lo.ToPtr(options.FromContext(ctx).ClusterName)}, + )) }) It("should not call CreateInstanceProfile or AddRoleToInstanceProfile when instance profile exists with correct role", func() { awsEnv.IAMAPI.InstanceProfiles = map[string]*iam.InstanceProfile{ @@ -105,7 +147,7 @@ var _ = Describe("NodeClass InstanceProfile Status Controller", func() { nodeClass.Spec.Role = "test-role" ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(1)) Expect(awsEnv.IAMAPI.InstanceProfiles[profileName].Roles).To(HaveLen(1)) @@ -113,23 +155,28 @@ var _ = Describe("NodeClass InstanceProfile Status Controller", func() { Expect(awsEnv.IAMAPI.CreateInstanceProfileBehavior.Calls()).To(BeZero()) Expect(awsEnv.IAMAPI.AddRoleToInstanceProfileBehavior.Calls()).To(BeZero()) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeInstanceProfileReady)).To(BeTrue()) }) It("should resolve the specified instance profile into the status when using instanceProfile field", func() { nodeClass.Spec.Role = "" nodeClass.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) Expect(nodeClass.Status.InstanceProfile).To(Equal(lo.FromPtr(nodeClass.Spec.InstanceProfile))) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeInstanceProfileReady)).To(BeTrue()) }) It("should not call the the IAM API when specifying an instance profile", func() { nodeClass.Spec.Role = "" nodeClass.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) Expect(awsEnv.IAMAPI.CreateInstanceProfileBehavior.Calls()).To(BeZero()) Expect(awsEnv.IAMAPI.AddRoleToInstanceProfileBehavior.Calls()).To(BeZero()) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeInstanceProfileReady)).To(BeTrue()) }) }) diff --git a/pkg/controllers/nodeclass/status/launchtemplate_test.go b/pkg/controllers/nodeclass/status/launchtemplate_test.go index 8d5bbb8457ea..8839a896d0f7 100644 --- a/pkg/controllers/nodeclass/status/launchtemplate_test.go +++ b/pkg/controllers/nodeclass/status/launchtemplate_test.go @@ -15,13 +15,11 @@ limitations under the License. package status_test import ( - _ "knative.dev/pkg/system/testing" - "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/aws/aws-sdk-go/service/eks" + "github.com/awslabs/operatorpkg/status" "github.com/samber/lo" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/test" . "github.com/onsi/ginkgo/v2" @@ -31,19 +29,20 @@ import ( var _ = Describe("NodeClass Launch Template CIDR Resolution Controller", func() { BeforeEach(func() { - nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ - SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ + nodeClass = test.EC2NodeClass(v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, - SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, - AMISelectorTerms: []v1beta1.AMISelectorTerm{ + AMIFamily: lo.ToPtr(v1.AMIFamilyCustom), + AMISelectorTerms: []v1.AMISelectorTerm{ { Tags: map[string]string{"*": "*"}, }, @@ -53,26 +52,29 @@ var _ = Describe("NodeClass Launch Template CIDR Resolution Controller", func() // Cluster CIDR will only be resolved once per lifetime of the launch template provider, reset to nil between tests awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(nil) }) - It("shouldn't resolve cluster CIDR for non-AL2023 NodeClasses", func() { - for _, family := range []string{ - v1beta1.AMIFamilyAL2, - v1beta1.AMIFamilyBottlerocket, - v1beta1.AMIFamilyUbuntu, - v1beta1.AMIFamilyWindows2019, - v1beta1.AMIFamilyWindows2022, - v1beta1.AMIFamilyCustom, - } { + DescribeTable( + "shouldn't resolve cluster CIDR for non-AL2023 NodeClasses", + func(family string, terms []v1.AMISelectorTerm) { nodeClass.Spec.AMIFamily = lo.ToPtr(family) + nodeClass.Spec.AMISelectorTerms = terms ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) Expect(awsEnv.LaunchTemplateProvider.ClusterCIDR.Load()).To(BeNil()) - } - }) + }, + Entry(v1.AMIFamilyAL2, v1.AMIFamilyAL2, []v1.AMISelectorTerm{{Alias: "al2@latest"}}), + Entry(v1.AMIFamilyBottlerocket, v1.AMIFamilyBottlerocket, []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}}), + Entry(v1.AMIFamilyWindows2019, v1.AMIFamilyWindows2019, []v1.AMISelectorTerm{{Alias: "windows2019@latest"}}), + Entry(v1.AMIFamilyWindows2022, v1.AMIFamilyWindows2022, []v1.AMISelectorTerm{{Alias: "windows2022@latest"}}), + Entry(v1.AMIFamilyCustom, v1.AMIFamilyCustom, []v1.AMISelectorTerm{{ID: "ami-12345"}}), + ) It("should resolve cluster CIDR for IPv4 clusters", func() { - nodeClass.Spec.AMIFamily = lo.ToPtr(v1beta1.AMIFamilyAL2023) + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2023@latest"}} ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) Expect(lo.FromPtr(awsEnv.LaunchTemplateProvider.ClusterCIDR.Load())).To(Equal("10.100.0.0/16")) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + Expect(nodeClass.StatusConditions().IsTrue(status.ConditionReady)).To(BeTrue()) }) It("should resolve cluster CIDR for IPv6 clusters", func() { awsEnv.EKSAPI.DescribeClusterBehavior.Output.Set(&eks.DescribeClusterOutput{ @@ -82,9 +84,12 @@ var _ = Describe("NodeClass Launch Template CIDR Resolution Controller", func() }, }, }) - nodeClass.Spec.AMIFamily = lo.ToPtr(v1beta1.AMIFamilyAL2023) + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2023@latest"}} ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) Expect(lo.FromPtr(awsEnv.LaunchTemplateProvider.ClusterCIDR.Load())).To(Equal("2001:db8::/64")) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + Expect(nodeClass.StatusConditions().IsTrue(status.ConditionReady)).To(BeTrue()) }) }) diff --git a/pkg/controllers/nodeclass/status/launchtemplate.go b/pkg/controllers/nodeclass/status/readiness.go similarity index 60% rename from pkg/controllers/nodeclass/status/launchtemplate.go rename to pkg/controllers/nodeclass/status/readiness.go index 7f8477b099fe..828ae098010b 100644 --- a/pkg/controllers/nodeclass/status/launchtemplate.go +++ b/pkg/controllers/nodeclass/status/readiness.go @@ -18,24 +18,27 @@ import ( "context" "fmt" - "github.com/samber/lo" - "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/awslabs/operatorpkg/status" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" ) -type LaunchTemplate struct { +type Readiness struct { launchTemplateProvider launchtemplate.Provider } -func (lt *LaunchTemplate) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (reconcile.Result, error) { - // A NodeClass that use AL2023 requires the cluster CIDR for launching nodes. +func (n Readiness) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconcile.Result, error) { + // A NodeClass that uses AL2023 requires the cluster CIDR for launching nodes. // To allow Karpenter to be used for Non-EKS clusters, resolving the Cluster CIDR // will not be done at startup but instead in a reconcile loop. - if lo.FromPtr(nodeClass.Spec.AMIFamily) == v1beta1.AMIFamilyAL2023 { - if err := lt.launchTemplateProvider.ResolveClusterCIDR(ctx); err != nil { - return reconcile.Result{}, fmt.Errorf("unable to detect the cluster CIDR, %w", err) + if nodeClass.AMIFamily() == v1.AMIFamilyAL2023 { + if err := n.launchTemplateProvider.ResolveClusterCIDR(ctx); err != nil { + nodeClass.StatusConditions().SetFalse(status.ConditionReady, "NodeClassNotReady", "Failed to detect the cluster CIDR") + return reconcile.Result{}, fmt.Errorf("failed to detect the cluster CIDR, %w", err) } } return reconcile.Result{}, nil diff --git a/pkg/controllers/nodeclass/status/readiness_test.go b/pkg/controllers/nodeclass/status/readiness_test.go new file mode 100644 index 000000000000..419732dbb6ba --- /dev/null +++ b/pkg/controllers/nodeclass/status/readiness_test.go @@ -0,0 +1,72 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package status_test + +import ( + "github.com/awslabs/operatorpkg/status" + "github.com/samber/lo" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + "github.com/aws/karpenter-provider-aws/pkg/test" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "sigs.k8s.io/karpenter/pkg/test/expectations" +) + +var _ = Describe("NodeClass Status Condition Controller", func() { + BeforeEach(func() { + nodeClass = test.EC2NodeClass(v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{"*": "*"}, + }, + }, + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"*": "*"}, + }, + }, + AMIFamily: lo.ToPtr(v1.AMIFamilyCustom), + AMISelectorTerms: []v1.AMISelectorTerm{ + { + Tags: map[string]string{"*": "*"}, + }, + }, + }, + }) + }) + It("should update status condition on nodeClass as Ready", func() { + ExpectApplied(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + Expect(nodeClass.Status.Conditions).To(HaveLen(5)) + Expect(nodeClass.StatusConditions().Get(status.ConditionReady).IsTrue()).To(BeTrue()) + }) + It("should update status condition as Not Ready", func() { + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"foo": "invalid"}, + }, + } + ExpectApplied(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + + Expect(nodeClass.StatusConditions().Get(status.ConditionReady).IsFalse()).To(BeTrue()) + Expect(nodeClass.StatusConditions().Get(status.ConditionReady).Message).To(Equal("SecurityGroupsReady=False")) + }) +}) diff --git a/pkg/controllers/nodeclass/status/securitygroup.go b/pkg/controllers/nodeclass/status/securitygroup.go index 378398ef04c0..f003b2dfd3ea 100644 --- a/pkg/controllers/nodeclass/status/securitygroup.go +++ b/pkg/controllers/nodeclass/status/securitygroup.go @@ -24,7 +24,7 @@ import ( "github.com/samber/lo" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/securitygroup" ) @@ -32,23 +32,25 @@ type SecurityGroup struct { securityGroupProvider securitygroup.Provider } -func (sg *SecurityGroup) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (reconcile.Result, error) { +func (sg *SecurityGroup) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconcile.Result, error) { securityGroups, err := sg.securityGroupProvider.List(ctx, nodeClass) if err != nil { - return reconcile.Result{}, err + return reconcile.Result{}, fmt.Errorf("getting security groups, %w", err) } if len(securityGroups) == 0 && len(nodeClass.Spec.SecurityGroupSelectorTerms) > 0 { nodeClass.Status.SecurityGroups = nil - return reconcile.Result{}, fmt.Errorf("no security groups exist given constraints") + nodeClass.StatusConditions().SetFalse(v1.ConditionTypeSecurityGroupsReady, "SecurityGroupsNotFound", "SecurityGroupSelector did not match any SecurityGroups") + return reconcile.Result{}, nil } sort.Slice(securityGroups, func(i, j int) bool { return *securityGroups[i].GroupId < *securityGroups[j].GroupId }) - nodeClass.Status.SecurityGroups = lo.Map(securityGroups, func(securityGroup *ec2.SecurityGroup, _ int) v1beta1.SecurityGroup { - return v1beta1.SecurityGroup{ + nodeClass.Status.SecurityGroups = lo.Map(securityGroups, func(securityGroup *ec2.SecurityGroup, _ int) v1.SecurityGroup { + return v1.SecurityGroup{ ID: *securityGroup.GroupId, Name: *securityGroup.GroupName, } }) + nodeClass.StatusConditions().SetTrue(v1.ConditionTypeSecurityGroupsReady) return reconcile.Result{RequeueAfter: 5 * time.Minute}, nil } diff --git a/pkg/controllers/nodeclass/status/securitygroup_test.go b/pkg/controllers/nodeclass/status/securitygroup_test.go index ad0589122921..3d78bf69ab65 100644 --- a/pkg/controllers/nodeclass/status/securitygroup_test.go +++ b/pkg/controllers/nodeclass/status/securitygroup_test.go @@ -15,10 +15,9 @@ limitations under the License. package status_test import ( - _ "knative.dev/pkg/system/testing" - "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/samber/lo" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/test" . "github.com/onsi/ginkgo/v2" @@ -28,19 +27,20 @@ import ( var _ = Describe("NodeClass Security Group Status Controller", func() { BeforeEach(func() { - nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ - SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ + nodeClass = test.EC2NodeClass(v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, - SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, - AMISelectorTerms: []v1beta1.AMISelectorTerm{ + AMIFamily: lo.ToPtr(v1.AMIFamilyCustom), + AMISelectorTerms: []v1.AMISelectorTerm{ { Tags: map[string]string{"*": "*"}, }, @@ -50,9 +50,9 @@ var _ = Describe("NodeClass Security Group Status Controller", func() { }) It("Should update EC2NodeClass status for Security Groups", func() { ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1beta1.SecurityGroup{ + Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1.SecurityGroup{ { ID: "sg-test1", Name: "securityGroup-test1", @@ -66,9 +66,10 @@ var _ = Describe("NodeClass Security Group Status Controller", func() { Name: "securityGroup-test3", }, })) + Expect(nodeClass.StatusConditions().Get(v1.ConditionTypeSecurityGroupsReady).IsTrue()).To(BeTrue()) }) It("Should resolve a valid selectors for Security Groups by tags", func() { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"Name": "test-security-group-1"}, }, @@ -77,9 +78,9 @@ var _ = Describe("NodeClass Security Group Status Controller", func() { }, } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1beta1.SecurityGroup{ + Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1.SecurityGroup{ { ID: "sg-test1", Name: "securityGroup-test1", @@ -89,28 +90,30 @@ var _ = Describe("NodeClass Security Group Status Controller", func() { Name: "securityGroup-test2", }, })) + Expect(nodeClass.StatusConditions().Get(v1.ConditionTypeSecurityGroupsReady).IsTrue()).To(BeTrue()) }) It("Should resolve a valid selectors for Security Groups by ids", func() { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { ID: "sg-test1", }, } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1beta1.SecurityGroup{ + Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1.SecurityGroup{ { ID: "sg-test1", Name: "securityGroup-test1", }, })) + Expect(nodeClass.StatusConditions().Get(v1.ConditionTypeSecurityGroupsReady).IsTrue()).To(BeTrue()) }) It("Should update Security Groups status when the Security Groups selector gets updated by tags", func() { ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1beta1.SecurityGroup{ + Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1.SecurityGroup{ { ID: "sg-test1", Name: "securityGroup-test1", @@ -125,7 +128,7 @@ var _ = Describe("NodeClass Security Group Status Controller", func() { }, })) - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"Name": "test-security-group-1"}, }, @@ -134,9 +137,9 @@ var _ = Describe("NodeClass Security Group Status Controller", func() { }, } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1beta1.SecurityGroup{ + Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1.SecurityGroup{ { ID: "sg-test1", Name: "securityGroup-test1", @@ -146,12 +149,13 @@ var _ = Describe("NodeClass Security Group Status Controller", func() { Name: "securityGroup-test2", }, })) + Expect(nodeClass.StatusConditions().Get(v1.ConditionTypeSecurityGroupsReady).IsTrue()).To(BeTrue()) }) It("Should update Security Groups status when the Security Groups selector gets updated by ids", func() { ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1beta1.SecurityGroup{ + Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1.SecurityGroup{ { ID: "sg-test1", Name: "securityGroup-test1", @@ -166,37 +170,39 @@ var _ = Describe("NodeClass Security Group Status Controller", func() { }, })) - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { ID: "sg-test1", }, } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1beta1.SecurityGroup{ + Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1.SecurityGroup{ { ID: "sg-test1", Name: "securityGroup-test1", }, })) + Expect(nodeClass.StatusConditions().Get(v1.ConditionTypeSecurityGroupsReady).IsTrue()).To(BeTrue()) }) It("Should not resolve a invalid selectors for Security Groups", func() { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{`foo`: `invalid`}, }, } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileFailed(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) Expect(nodeClass.Status.SecurityGroups).To(BeNil()) + Expect(nodeClass.StatusConditions().Get(v1.ConditionTypeSecurityGroupsReady).IsFalse()).To(BeTrue()) }) It("Should not resolve a invalid selectors for an updated Security Groups selector", func() { ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1beta1.SecurityGroup{ + Expect(nodeClass.Status.SecurityGroups).To(Equal([]v1.SecurityGroup{ { ID: "sg-test1", Name: "securityGroup-test1", @@ -211,14 +217,15 @@ var _ = Describe("NodeClass Security Group Status Controller", func() { }, })) - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{`foo`: `invalid`}, }, } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileFailed(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) Expect(nodeClass.Status.SecurityGroups).To(BeNil()) + Expect(nodeClass.StatusConditions().Get(v1.ConditionTypeSecurityGroupsReady).IsFalse()).To(BeTrue()) }) }) diff --git a/pkg/controllers/nodeclass/status/subnet.go b/pkg/controllers/nodeclass/status/subnet.go index 861131fa8cbc..4e71dd0384a1 100644 --- a/pkg/controllers/nodeclass/status/subnet.go +++ b/pkg/controllers/nodeclass/status/subnet.go @@ -24,7 +24,7 @@ import ( "github.com/samber/lo" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/subnet" ) @@ -32,14 +32,15 @@ type Subnet struct { subnetProvider subnet.Provider } -func (s *Subnet) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (reconcile.Result, error) { +func (s *Subnet) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconcile.Result, error) { subnets, err := s.subnetProvider.List(ctx, nodeClass) if err != nil { - return reconcile.Result{}, err + return reconcile.Result{}, fmt.Errorf("getting subnets, %w", err) } if len(subnets) == 0 { nodeClass.Status.Subnets = nil - return reconcile.Result{}, fmt.Errorf("no subnets exist given constraints %v", nodeClass.Spec.SubnetSelectorTerms) + nodeClass.StatusConditions().SetFalse(v1.ConditionTypeSubnetsReady, "SubnetsNotFound", "SubnetSelector did not match any Subnets") + return reconcile.Result{}, nil } sort.Slice(subnets, func(i, j int) bool { if int(*subnets[i].AvailableIpAddressCount) != int(*subnets[j].AvailableIpAddressCount) { @@ -47,12 +48,13 @@ func (s *Subnet) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) } return *subnets[i].SubnetId < *subnets[j].SubnetId }) - nodeClass.Status.Subnets = lo.Map(subnets, func(ec2subnet *ec2.Subnet, _ int) v1beta1.Subnet { - return v1beta1.Subnet{ - ID: *ec2subnet.SubnetId, - Zone: *ec2subnet.AvailabilityZone, + nodeClass.Status.Subnets = lo.Map(subnets, func(ec2subnet *ec2.Subnet, _ int) v1.Subnet { + return v1.Subnet{ + ID: *ec2subnet.SubnetId, + Zone: *ec2subnet.AvailabilityZone, + ZoneID: *ec2subnet.AvailabilityZoneId, } }) - - return reconcile.Result{RequeueAfter: 5 * time.Minute}, nil + nodeClass.StatusConditions().SetTrue(v1.ConditionTypeSubnetsReady) + return reconcile.Result{RequeueAfter: time.Minute}, nil } diff --git a/pkg/controllers/nodeclass/status/subnet_test.go b/pkg/controllers/nodeclass/status/subnet_test.go index ba37f1d3b11a..21a11493614d 100644 --- a/pkg/controllers/nodeclass/status/subnet_test.go +++ b/pkg/controllers/nodeclass/status/subnet_test.go @@ -15,13 +15,11 @@ limitations under the License. package status_test import ( - _ "knative.dev/pkg/system/testing" - "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/samber/lo" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/test" . "github.com/onsi/ginkgo/v2" @@ -31,19 +29,20 @@ import ( var _ = Describe("NodeClass Subnet Status Controller", func() { BeforeEach(func() { - nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ - SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ + nodeClass = test.EC2NodeClass(v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, - SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, - AMISelectorTerms: []v1beta1.AMISelectorTerm{ + AMIFamily: lo.ToPtr(v1.AMIFamilyCustom), + AMISelectorTerms: []v1.AMISelectorTerm{ { Tags: map[string]string{"*": "*"}, }, @@ -53,53 +52,62 @@ var _ = Describe("NodeClass Subnet Status Controller", func() { }) It("Should update EC2NodeClass status for Subnets", func() { ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.Subnets).To(Equal([]v1beta1.Subnet{ + Expect(nodeClass.Status.Subnets).To(Equal([]v1.Subnet{ { - ID: "subnet-test1", - Zone: "test-zone-1a", + ID: "subnet-test1", + Zone: "test-zone-1a", + ZoneID: "tstz1-1a", }, { - ID: "subnet-test2", - Zone: "test-zone-1b", + ID: "subnet-test2", + Zone: "test-zone-1b", + ZoneID: "tstz1-1b", }, { - ID: "subnet-test3", - Zone: "test-zone-1c", + ID: "subnet-test3", + Zone: "test-zone-1c", + ZoneID: "tstz1-1c", }, { - ID: "subnet-test4", - Zone: "test-zone-1a-local", + ID: "subnet-test4", + Zone: "test-zone-1a-local", + ZoneID: "tstz1-1alocal", }, })) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeSubnetsReady)).To(BeTrue()) }) It("Should have the correct ordering for the Subnets", func() { awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ - {SubnetId: aws.String("subnet-test1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(20)}, - {SubnetId: aws.String("subnet-test2"), AvailabilityZone: aws.String("test-zone-1b"), AvailableIpAddressCount: aws.Int64(100)}, - {SubnetId: aws.String("subnet-test3"), AvailabilityZone: aws.String("test-zone-1c"), AvailableIpAddressCount: aws.Int64(50)}, + {SubnetId: aws.String("subnet-test1"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(20)}, + {SubnetId: aws.String("subnet-test2"), AvailabilityZone: aws.String("test-zone-1b"), AvailabilityZoneId: aws.String("tstz1-1b"), AvailableIpAddressCount: aws.Int64(100)}, + {SubnetId: aws.String("subnet-test3"), AvailabilityZone: aws.String("test-zone-1c"), AvailabilityZoneId: aws.String("tstz1-1c"), AvailableIpAddressCount: aws.Int64(50)}, }}) ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.Subnets).To(Equal([]v1beta1.Subnet{ + Expect(nodeClass.Status.Subnets).To(Equal([]v1.Subnet{ { - ID: "subnet-test2", - Zone: "test-zone-1b", + ID: "subnet-test2", + Zone: "test-zone-1b", + ZoneID: "tstz1-1b", }, { - ID: "subnet-test3", - Zone: "test-zone-1c", + ID: "subnet-test3", + Zone: "test-zone-1c", + ZoneID: "tstz1-1c", }, { - ID: "subnet-test1", - Zone: "test-zone-1a", + ID: "subnet-test1", + Zone: "test-zone-1a", + ZoneID: "tstz1-1a", }, })) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeSubnetsReady)).To(BeTrue()) }) It("Should resolve a valid selectors for Subnet by tags", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { Tags: map[string]string{`Name`: `test-subnet-1`}, }, @@ -108,59 +116,68 @@ var _ = Describe("NodeClass Subnet Status Controller", func() { }, } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.Subnets).To(Equal([]v1beta1.Subnet{ + Expect(nodeClass.Status.Subnets).To(Equal([]v1.Subnet{ { - ID: "subnet-test1", - Zone: "test-zone-1a", + ID: "subnet-test1", + Zone: "test-zone-1a", + ZoneID: "tstz1-1a", }, { - ID: "subnet-test2", - Zone: "test-zone-1b", + ID: "subnet-test2", + Zone: "test-zone-1b", + ZoneID: "tstz1-1b", }, })) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeSubnetsReady)).To(BeTrue()) }) It("Should resolve a valid selectors for Subnet by ids", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { ID: "subnet-test1", }, } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.Subnets).To(Equal([]v1beta1.Subnet{ + Expect(nodeClass.Status.Subnets).To(Equal([]v1.Subnet{ { - ID: "subnet-test1", - Zone: "test-zone-1a", + ID: "subnet-test1", + Zone: "test-zone-1a", + ZoneID: "tstz1-1a", }, })) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeSubnetsReady)).To(BeTrue()) }) It("Should update Subnet status when the Subnet selector gets updated by tags", func() { ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.Subnets).To(Equal([]v1beta1.Subnet{ + Expect(nodeClass.Status.Subnets).To(Equal([]v1.Subnet{ { - ID: "subnet-test1", - Zone: "test-zone-1a", + ID: "subnet-test1", + Zone: "test-zone-1a", + ZoneID: "tstz1-1a", }, { - ID: "subnet-test2", - Zone: "test-zone-1b", + ID: "subnet-test2", + Zone: "test-zone-1b", + ZoneID: "tstz1-1b", }, { - ID: "subnet-test3", - Zone: "test-zone-1c", + ID: "subnet-test3", + Zone: "test-zone-1c", + ZoneID: "tstz1-1c", }, { - ID: "subnet-test4", - Zone: "test-zone-1a-local", + ID: "subnet-test4", + Zone: "test-zone-1a-local", + ZoneID: "tstz1-1alocal", }, })) - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { Tags: map[string]string{ "Name": "test-subnet-1", @@ -173,99 +190,114 @@ var _ = Describe("NodeClass Subnet Status Controller", func() { }, } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.Subnets).To(Equal([]v1beta1.Subnet{ + Expect(nodeClass.Status.Subnets).To(Equal([]v1.Subnet{ { - ID: "subnet-test1", - Zone: "test-zone-1a", + ID: "subnet-test1", + Zone: "test-zone-1a", + ZoneID: "tstz1-1a", }, { - ID: "subnet-test2", - Zone: "test-zone-1b", + ID: "subnet-test2", + Zone: "test-zone-1b", + ZoneID: "tstz1-1b", }, })) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeSubnetsReady)).To(BeTrue()) }) It("Should update Subnet status when the Subnet selector gets updated by ids", func() { ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.Subnets).To(Equal([]v1beta1.Subnet{ + Expect(nodeClass.Status.Subnets).To(Equal([]v1.Subnet{ { - ID: "subnet-test1", - Zone: "test-zone-1a", + ID: "subnet-test1", + Zone: "test-zone-1a", + ZoneID: "tstz1-1a", }, { - ID: "subnet-test2", - Zone: "test-zone-1b", + ID: "subnet-test2", + Zone: "test-zone-1b", + ZoneID: "tstz1-1b", }, { - ID: "subnet-test3", - Zone: "test-zone-1c", + ID: "subnet-test3", + Zone: "test-zone-1c", + ZoneID: "tstz1-1c", }, { - ID: "subnet-test4", - Zone: "test-zone-1a-local", + ID: "subnet-test4", + Zone: "test-zone-1a-local", + ZoneID: "tstz1-1alocal", }, })) - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { ID: "subnet-test1", }, } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.Subnets).To(Equal([]v1beta1.Subnet{ + Expect(nodeClass.Status.Subnets).To(Equal([]v1.Subnet{ { - ID: "subnet-test1", - Zone: "test-zone-1a", + ID: "subnet-test1", + Zone: "test-zone-1a", + ZoneID: "tstz1-1a", }, })) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeSubnetsReady)).To(BeTrue()) }) It("Should not resolve a invalid selectors for Subnet", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { Tags: map[string]string{`foo`: `invalid`}, }, } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileFailed(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) Expect(nodeClass.Status.Subnets).To(BeNil()) + Expect(nodeClass.StatusConditions().Get(v1.ConditionTypeSubnetsReady).IsFalse()).To(BeTrue()) }) It("Should not resolve a invalid selectors for an updated subnet selector", func() { ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(nodeClass.Status.Subnets).To(Equal([]v1beta1.Subnet{ + Expect(nodeClass.Status.Subnets).To(Equal([]v1.Subnet{ { - ID: "subnet-test1", - Zone: "test-zone-1a", + ID: "subnet-test1", + Zone: "test-zone-1a", + ZoneID: "tstz1-1a", }, { - ID: "subnet-test2", - Zone: "test-zone-1b", + ID: "subnet-test2", + Zone: "test-zone-1b", + ZoneID: "tstz1-1b", }, { - ID: "subnet-test3", - Zone: "test-zone-1c", + ID: "subnet-test3", + Zone: "test-zone-1c", + ZoneID: "tstz1-1c", }, { - ID: "subnet-test4", - Zone: "test-zone-1a-local", + ID: "subnet-test4", + Zone: "test-zone-1a-local", + ZoneID: "tstz1-1alocal", }, })) - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { Tags: map[string]string{`foo`: `invalid`}, }, } ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileFailed(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) Expect(nodeClass.Status.Subnets).To(BeNil()) + Expect(nodeClass.StatusConditions().Get(v1.ConditionTypeSubnetsReady).IsFalse()).To(BeTrue()) }) }) diff --git a/pkg/controllers/nodeclass/status/suite_test.go b/pkg/controllers/nodeclass/status/suite_test.go index 545c9ba6cfd5..3eb2b1152ec5 100644 --- a/pkg/controllers/nodeclass/status/suite_test.go +++ b/pkg/controllers/nodeclass/status/suite_test.go @@ -18,30 +18,28 @@ import ( "context" "testing" - _ "knative.dev/pkg/system/testing" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/test" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) var ctx context.Context var env *coretest.Environment var awsEnv *test.Environment -var nodeClass *v1beta1.EC2NodeClass -var statusController corecontroller.Controller +var nodeClass *v1.EC2NodeClass +var statusController *status.Controller func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -50,7 +48,7 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...), coretest.WithFieldIndexers(test.EC2NodeClassFieldIndexer(ctx))) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...), coretest.WithFieldIndexers(test.EC2NodeClassFieldIndexer(ctx))) ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) awsEnv = test.NewEnvironment(ctx, env) diff --git a/pkg/controllers/nodeclass/termination/controller.go b/pkg/controllers/nodeclass/termination/controller.go index 39a740357700..5785da72a43c 100644 --- a/pkg/controllers/nodeclass/termination/controller.go +++ b/pkg/controllers/nodeclass/termination/controller.go @@ -20,14 +20,13 @@ import ( "time" "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/karpenter/pkg/operator/injection" "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" "github.com/samber/lo" - "golang.org/x/time/rate" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/workqueue" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,16 +38,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "github.com/awslabs/operatorpkg/reasonable" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/events" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" ) -var _ corecontroller.FinalizingTypedController[*v1beta1.EC2NodeClass] = (*Controller)(nil) - type Controller struct { kubeClient client.Client recorder events.Recorder @@ -57,31 +54,36 @@ type Controller struct { } func NewController(kubeClient client.Client, recorder events.Recorder, - instanceProfileProvider instanceprofile.Provider, launchTemplateProvider launchtemplate.Provider) corecontroller.Controller { + instanceProfileProvider instanceprofile.Provider, launchTemplateProvider launchtemplate.Provider) *Controller { - return corecontroller.Typed[*v1beta1.EC2NodeClass](kubeClient, &Controller{ + return &Controller{ kubeClient: kubeClient, recorder: recorder, instanceProfileProvider: instanceProfileProvider, launchTemplateProvider: launchTemplateProvider, - }) + } } -func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (reconcile.Result, error) { +func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconcile.Result, error) { + ctx = injection.WithControllerName(ctx, "nodeclass.termination") + + if !nodeClass.GetDeletionTimestamp().IsZero() { + return c.finalize(ctx, nodeClass) + } return reconcile.Result{}, nil } -func (c *Controller) Finalize(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (reconcile.Result, error) { +func (c *Controller) finalize(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconcile.Result, error) { stored := nodeClass.DeepCopy() - if !controllerutil.ContainsFinalizer(nodeClass, v1beta1.TerminationFinalizer) { + if !controllerutil.ContainsFinalizer(nodeClass, v1.TerminationFinalizer) { return reconcile.Result{}, nil } - nodeClaimList := &corev1beta1.NodeClaimList{} + nodeClaimList := &karpv1.NodeClaimList{} if err := c.kubeClient.List(ctx, nodeClaimList, client.MatchingFields{"spec.nodeClassRef.name": nodeClass.Name}); err != nil { return reconcile.Result{}, fmt.Errorf("listing nodeclaims that are using nodeclass, %w", err) } if len(nodeClaimList.Items) > 0 { - c.recorder.Publish(WaitingOnNodeClaimTerminationEvent(nodeClass, lo.Map(nodeClaimList.Items, func(nc corev1beta1.NodeClaim, _ int) string { return nc.Name }))) + c.recorder.Publish(WaitingOnNodeClaimTerminationEvent(nodeClass, lo.Map(nodeClaimList.Items, func(nc karpv1.NodeClaim, _ int) string { return nc.Name }))) return reconcile.Result{RequeueAfter: time.Minute * 10}, nil // periodically fire the event } if nodeClass.Spec.Role != "" { @@ -92,7 +94,7 @@ func (c *Controller) Finalize(ctx context.Context, nodeClass *v1beta1.EC2NodeCla if err := c.launchTemplateProvider.DeleteAll(ctx, nodeClass); err != nil { return reconcile.Result{}, fmt.Errorf("deleting launch templates, %w", err) } - controllerutil.RemoveFinalizer(nodeClass, v1beta1.TerminationFinalizer) + controllerutil.RemoveFinalizer(nodeClass, v1.TerminationFinalizer) if !equality.Semantic.DeepEqual(stored, nodeClass) { // We call Update() here rather than Patch() because patching a list with a JSON merge patch // can cause races due to the fact that it fully replaces the list on a change @@ -107,18 +109,14 @@ func (c *Controller) Finalize(ctx context.Context, nodeClass *v1beta1.EC2NodeCla return reconcile.Result{}, nil } -func (c *Controller) Name() string { - return "nodeclass.termination" -} - -func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { - return corecontroller.Adapt(controllerruntime. - NewControllerManagedBy(m). - For(&v1beta1.EC2NodeClass{}). +func (c *Controller) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("nodeclass.termination"). + For(&v1.EC2NodeClass{}). Watches( - &corev1beta1.NodeClaim{}, + &karpv1.NodeClaim{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, o client.Object) []reconcile.Request { - nc := o.(*corev1beta1.NodeClaim) + nc := o.(*karpv1.NodeClaim) if nc.Spec.NodeClassRef == nil { return nil } @@ -132,11 +130,8 @@ func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontrolle }), ). WithOptions(controller.Options{ - RateLimiter: workqueue.NewMaxOfRateLimiter( - workqueue.NewItemExponentialFailureRateLimiter(100*time.Millisecond, 1*time.Minute), - // 10 qps, 100 bucket size - &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, - ), + RateLimiter: reasonable.RateLimiter(), MaxConcurrentReconciles: 10, - })) + }). + Complete(reconcile.AsReconciler(m.GetClient(), c)) } diff --git a/pkg/controllers/nodeclass/termination/events.go b/pkg/controllers/nodeclass/termination/events.go index 8da543ed2bd8..3823e871c806 100644 --- a/pkg/controllers/nodeclass/termination/events.go +++ b/pkg/controllers/nodeclass/termination/events.go @@ -17,18 +17,18 @@ package termination import ( "fmt" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/karpenter/pkg/events" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/utils" ) -func WaitingOnNodeClaimTerminationEvent(nodeClass *v1beta1.EC2NodeClass, names []string) events.Event { +func WaitingOnNodeClaimTerminationEvent(nodeClass *v1.EC2NodeClass, names []string) events.Event { return events.Event{ InvolvedObject: nodeClass, - Type: v1.EventTypeNormal, + Type: corev1.EventTypeNormal, Reason: "WaitingOnNodeClaimTermination", Message: fmt.Sprintf("Waiting on NodeClaim termination for %s", utils.PrettySlice(names, 5)), DedupeValues: []string{string(nodeClass.UID)}, diff --git a/pkg/controllers/nodeclass/termination/suite_test.go b/pkg/controllers/nodeclass/termination/suite_test.go index 7b707f516039..c24f42121d6f 100644 --- a/pkg/controllers/nodeclass/termination/suite_test.go +++ b/pkg/controllers/nodeclass/termination/suite_test.go @@ -20,24 +20,22 @@ import ( "testing" "time" - "github.com/samber/lo" - "k8s.io/client-go/tools/record" - _ "knative.dev/pkg/system/testing" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/iam" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "github.com/awslabs/operatorpkg/object" + "github.com/samber/lo" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/events" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/termination" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" @@ -45,14 +43,14 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) var ctx context.Context var env *coretest.Environment var awsEnv *test.Environment -var terminationController corecontroller.Controller +var terminationController *termination.Controller func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -61,7 +59,7 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...), coretest.WithFieldIndexers(test.EC2NodeClassFieldIndexer(ctx))) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...), coretest.WithFieldIndexers(test.EC2NodeClassFieldIndexer(ctx))) ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) awsEnv = test.NewEnvironment(ctx, env) @@ -83,22 +81,23 @@ var _ = AfterEach(func() { }) var _ = Describe("NodeClass Termination", func() { - var nodeClass *v1beta1.EC2NodeClass + var nodeClass *v1.EC2NodeClass var profileName string BeforeEach(func() { - nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ - SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ + nodeClass = test.EC2NodeClass(v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, - SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"*": "*"}, }, }, - AMISelectorTerms: []v1beta1.AMISelectorTerm{ + AMIFamily: lo.ToPtr(v1.AMIFamilyCustom), + AMISelectorTerms: []v1.AMISelectorTerm{ { Tags: map[string]string{"*": "*"}, }, @@ -113,13 +112,13 @@ var _ = Describe("NodeClass Termination", func() { awsEnv.EC2API.LaunchTemplates.Store(launchTemplateName, &ec2.LaunchTemplate{LaunchTemplateName: launchTemplateName, LaunchTemplateId: aws.String(fake.LaunchTemplateID()), Tags: []*ec2.Tag{&ec2.Tag{Key: aws.String("karpenter.k8s.aws/cluster"), Value: aws.String("test-cluster")}}}) _, ok := awsEnv.EC2API.LaunchTemplates.Load(launchTemplateName) Expect(ok).To(BeTrue()) - controllerutil.AddFinalizer(nodeClass, v1beta1.TerminationFinalizer) + controllerutil.AddFinalizer(nodeClass, v1.TerminationFinalizer) ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(env.Client.Delete(ctx, nodeClass)).To(Succeed()) awsEnv.EC2API.NextError.Set(fmt.Errorf("delete Launch Template Error")) - ExpectReconcileFailed(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + _ = ExpectObjectReconcileFailed(ctx, env.Client, terminationController, nodeClass) ExpectExists(ctx, env.Client, nodeClass) }) It("should not delete the launch template not associated with the nodeClass", func() { @@ -127,12 +126,12 @@ var _ = Describe("NodeClass Termination", func() { awsEnv.EC2API.LaunchTemplates.Store(launchTemplateName, &ec2.LaunchTemplate{LaunchTemplateName: launchTemplateName, LaunchTemplateId: aws.String(fake.LaunchTemplateID()), Tags: []*ec2.Tag{&ec2.Tag{Key: aws.String("karpenter.k8s.aws/cluster"), Value: aws.String("test-cluster")}}}) _, ok := awsEnv.EC2API.LaunchTemplates.Load(launchTemplateName) Expect(ok).To(BeTrue()) - controllerutil.AddFinalizer(nodeClass, v1beta1.TerminationFinalizer) + controllerutil.AddFinalizer(nodeClass, v1.TerminationFinalizer) ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(env.Client.Delete(ctx, nodeClass)).To(Succeed()) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) _, ok = awsEnv.EC2API.LaunchTemplates.Load(launchTemplateName) Expect(ok).To(BeTrue()) ExpectNotFound(ctx, env.Client, nodeClass) @@ -146,12 +145,12 @@ var _ = Describe("NodeClass Termination", func() { Expect(ok).To(BeTrue()) _, ok = awsEnv.EC2API.LaunchTemplates.Load(ltName2) Expect(ok).To(BeTrue()) - controllerutil.AddFinalizer(nodeClass, v1beta1.TerminationFinalizer) + controllerutil.AddFinalizer(nodeClass, v1.TerminationFinalizer) ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(env.Client.Delete(ctx, nodeClass)).To(Succeed()) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) _, ok = awsEnv.EC2API.LaunchTemplates.Load(ltName1) Expect(ok).To(BeFalse()) _, ok = awsEnv.EC2API.LaunchTemplates.Load(ltName2) @@ -170,13 +169,13 @@ var _ = Describe("NodeClass Termination", func() { }, }, } - controllerutil.AddFinalizer(nodeClass, v1beta1.TerminationFinalizer) + controllerutil.AddFinalizer(nodeClass, v1.TerminationFinalizer) ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(1)) Expect(env.Client.Delete(ctx, nodeClass)).To(Succeed()) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(0)) ExpectNotFound(ctx, env.Client, nodeClass) }) @@ -186,33 +185,35 @@ var _ = Describe("NodeClass Termination", func() { InstanceProfileName: aws.String(profileName), }, } - controllerutil.AddFinalizer(nodeClass, v1beta1.TerminationFinalizer) + controllerutil.AddFinalizer(nodeClass, v1.TerminationFinalizer) ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(1)) Expect(env.Client.Delete(ctx, nodeClass)).To(Succeed()) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(0)) ExpectNotFound(ctx, env.Client, nodeClass) }) It("should succeed to delete the NodeClass when the instance profile doesn't exist", func() { Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(0)) - controllerutil.AddFinalizer(nodeClass, v1beta1.TerminationFinalizer) + controllerutil.AddFinalizer(nodeClass, v1.TerminationFinalizer) ExpectApplied(ctx, env.Client, nodeClass) Expect(env.Client.Delete(ctx, nodeClass)).To(Succeed()) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(0)) ExpectNotFound(ctx, env.Client, nodeClass) }) It("should not delete the EC2NodeClass until all associated NodeClaims are terminated", func() { - var nodeClaims []*corev1beta1.NodeClaim + var nodeClaims []*karpv1.NodeClaim for i := 0; i < 2; i++ { - nc := coretest.NodeClaim(corev1beta1.NodeClaim{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nc := coretest.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) @@ -230,13 +231,13 @@ var _ = Describe("NodeClass Termination", func() { }, }, } - controllerutil.AddFinalizer(nodeClass, v1beta1.TerminationFinalizer) + controllerutil.AddFinalizer(nodeClass, v1.TerminationFinalizer) ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(1)) Expect(env.Client.Delete(ctx, nodeClass)).To(Succeed()) - res := ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + res := ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(res.RequeueAfter).To(Equal(time.Minute * 10)) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(1)) ExpectExists(ctx, env.Client, nodeClass) @@ -244,7 +245,7 @@ var _ = Describe("NodeClass Termination", func() { // Delete one of the NodeClaims // The NodeClass should still not delete ExpectDeleted(ctx, env.Client, nodeClaims[0]) - res = ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + res = ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(res.RequeueAfter).To(Equal(time.Minute * 10)) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(1)) ExpectExists(ctx, env.Client, nodeClass) @@ -252,7 +253,7 @@ var _ = Describe("NodeClass Termination", func() { // Delete the last NodeClaim // The NodeClass should now delete ExpectDeleted(ctx, env.Client, nodeClaims[1]) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(0)) ExpectNotFound(ctx, env.Client, nodeClass) }) @@ -270,13 +271,13 @@ var _ = Describe("NodeClass Termination", func() { } nodeClass.Spec.Role = "" nodeClass.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") - controllerutil.AddFinalizer(nodeClass, v1beta1.TerminationFinalizer) + controllerutil.AddFinalizer(nodeClass, v1.TerminationFinalizer) ExpectApplied(ctx, env.Client, nodeClass) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(1)) Expect(env.Client.Delete(ctx, nodeClass)).To(Succeed()) - ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(nodeClass)) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) Expect(awsEnv.IAMAPI.InstanceProfiles).To(HaveLen(1)) ExpectNotFound(ctx, env.Client, nodeClass) diff --git a/pkg/controllers/providers/instancetype/controller.go b/pkg/controllers/providers/instancetype/controller.go new file mode 100644 index 000000000000..38537ab88007 --- /dev/null +++ b/pkg/controllers/providers/instancetype/controller.go @@ -0,0 +1,68 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instancetype + +import ( + "context" + "fmt" + "time" + + "github.com/awslabs/operatorpkg/singleton" + lop "github.com/samber/lo/parallel" + "go.uber.org/multierr" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/karpenter/pkg/operator/injection" + + "github.com/aws/karpenter-provider-aws/pkg/providers/instancetype" +) + +type Controller struct { + instancetypeProvider instancetype.Provider +} + +func NewController(instancetypeProvider instancetype.Provider) *Controller { + return &Controller{ + instancetypeProvider: instancetypeProvider, + } +} + +func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) { + ctx = injection.WithControllerName(ctx, "providers.instancetype") + + work := []func(ctx context.Context) error{ + c.instancetypeProvider.UpdateInstanceTypes, + c.instancetypeProvider.UpdateInstanceTypeOfferings, + } + errs := make([]error, len(work)) + lop.ForEach(work, func(f func(ctx context.Context) error, i int) { + if err := f(ctx); err != nil { + errs[i] = err + } + }) + if err := multierr.Combine(errs...); err != nil { + return reconcile.Result{}, fmt.Errorf("updating instancetype, %w", err) + } + return reconcile.Result{RequeueAfter: 12 * time.Hour}, nil +} + +func (c *Controller) Register(_ context.Context, m manager.Manager) error { + // Includes a default exponential failure rate limiter of base: time.Millisecond, and max: 1000*time.Second + return controllerruntime.NewControllerManagedBy(m). + Named("providers.instancetype"). + WatchesRawSource(singleton.Source()). + Complete(singleton.AsReconciler(c)) +} diff --git a/pkg/controllers/providers/instancetype/suite_test.go b/pkg/controllers/providers/instancetype/suite_test.go new file mode 100644 index 000000000000..7f209fc15203 --- /dev/null +++ b/pkg/controllers/providers/instancetype/suite_test.go @@ -0,0 +1,171 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instancetype_test + +import ( + "context" + "testing" + + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + + corev1 "k8s.io/api/core/v1" + coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" + coretest "sigs.k8s.io/karpenter/pkg/test" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/samber/lo" + + "github.com/aws/karpenter-provider-aws/pkg/apis" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + controllersinstancetype "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/instancetype" + "github.com/aws/karpenter-provider-aws/pkg/fake" + "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/test" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" +) + +var ctx context.Context +var stop context.CancelFunc +var env *coretest.Environment +var awsEnv *test.Environment +var controller *controllersinstancetype.Controller + +func TestAWS(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "InstanceType") +} + +var _ = BeforeSuite(func() { + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) + ctx, stop = context.WithCancel(ctx) + awsEnv = test.NewEnvironment(ctx, env) + controller = controllersinstancetype.NewController(awsEnv.InstanceTypesProvider) +}) + +var _ = AfterSuite(func() { + stop() + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = BeforeEach(func() { + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) + + awsEnv.Reset() +}) + +var _ = AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) +}) + +var _ = Describe("InstanceType", func() { + It("should update instance type date with response from the DescribeInstanceTypes API", func() { + ec2InstanceTypes := fake.MakeInstances() + ec2Offerings := fake.MakeInstanceOfferings(ec2InstanceTypes) + awsEnv.EC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{ + InstanceTypes: ec2InstanceTypes, + }) + awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{ + InstanceTypeOfferings: ec2Offerings, + }) + + ExpectSingletonReconciled(ctx, controller) + instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, &v1.KubeletConfiguration{}, &v1.EC2NodeClass{ + Status: v1.EC2NodeClassStatus{ + Subnets: []v1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + }, + }, + }) + Expect(err).To(BeNil()) + for i := range instanceTypes { + Expect(instanceTypes[i].Name).To(Equal(lo.FromPtr(ec2InstanceTypes[i].InstanceType))) + } + }) + It("should update instance type offering date with response from the DescribeInstanceTypesOfferings API", func() { + ec2InstanceTypes := fake.MakeInstances() + ec2Offerings := fake.MakeInstanceOfferings(ec2InstanceTypes) + awsEnv.EC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{ + InstanceTypes: ec2InstanceTypes, + }) + awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{ + InstanceTypeOfferings: ec2Offerings, + }) + + ExpectSingletonReconciled(ctx, controller) + instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, &v1.KubeletConfiguration{}, &v1.EC2NodeClass{ + Status: v1.EC2NodeClassStatus{ + Subnets: []v1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + }, + }, + }) + Expect(err).To(BeNil()) + + Expect(len(instanceTypes)).To(BeNumerically("==", len(ec2InstanceTypes))) + for x := range instanceTypes { + offering, found := lo.Find(ec2Offerings, func(off *ec2.InstanceTypeOffering) bool { + return instanceTypes[x].Name == lo.FromPtr(off.InstanceType) + }) + Expect(found).To(BeTrue()) + for y := range instanceTypes[x].Offerings { + Expect(instanceTypes[x].Offerings[y].Requirements.Get(corev1.LabelTopologyZone).Any()).To(Equal(lo.FromPtr(offering.Location))) + } + } + }) + It("should not update instance type date with response from the DescribeInstanceTypes API is empty", func() { + awsEnv.EC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{}) + awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{}) + ExpectSingletonReconciled(ctx, controller) + _, err := awsEnv.InstanceTypesProvider.List(ctx, &v1.KubeletConfiguration{}, &v1.EC2NodeClass{}) + Expect(err).ToNot(BeNil()) + }) + It("should not update instance type offering date with response from the DescribeInstanceTypesOfferings API", func() { + awsEnv.EC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{}) + awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{}) + ExpectSingletonReconciled(ctx, controller) + _, err := awsEnv.InstanceTypesProvider.List(ctx, &v1.KubeletConfiguration{}, &v1.EC2NodeClass{}) + Expect(err).ToNot(BeNil()) + }) +}) diff --git a/pkg/controllers/pricing/controller.go b/pkg/controllers/providers/pricing/controller.go similarity index 73% rename from pkg/controllers/pricing/controller.go rename to pkg/controllers/providers/pricing/controller.go index 065e41130d03..24daae768bc6 100644 --- a/pkg/controllers/pricing/controller.go +++ b/pkg/controllers/providers/pricing/controller.go @@ -19,12 +19,13 @@ import ( "fmt" "time" + "github.com/awslabs/operatorpkg/singleton" lop "github.com/samber/lo/parallel" "go.uber.org/multierr" + controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "sigs.k8s.io/karpenter/pkg/operator/controller" + "sigs.k8s.io/karpenter/pkg/operator/injection" "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" ) @@ -39,7 +40,9 @@ func NewController(pricingProvider pricing.Provider) *Controller { } } -func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { +func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) { + ctx = injection.WithControllerName(ctx, "providers.pricing") + work := []func(ctx context.Context) error{ c.pricingProvider.UpdateSpotPricing, c.pricingProvider.UpdateOnDemandPricing, @@ -56,10 +59,9 @@ func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconc return reconcile.Result{RequeueAfter: 12 * time.Hour}, nil } -func (c *Controller) Name() string { - return "pricing" -} - -func (c *Controller) Builder(_ context.Context, m manager.Manager) controller.Builder { - return controller.NewSingletonManagedBy(m) +func (c *Controller) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("providers.pricing"). + WatchesRawSource(singleton.Source()). + Complete(singleton.AsReconciler(c)) } diff --git a/pkg/controllers/pricing/suite_test.go b/pkg/controllers/providers/pricing/suite_test.go similarity index 93% rename from pkg/controllers/pricing/suite_test.go rename to pkg/controllers/providers/pricing/suite_test.go index 382a398a2088..86edd6c5eb4c 100644 --- a/pkg/controllers/pricing/suite_test.go +++ b/pkg/controllers/providers/pricing/suite_test.go @@ -20,17 +20,17 @@ import ( "testing" "time" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" awspricing "github.com/aws/aws-sdk-go/service/pricing" "github.com/samber/lo" - "k8s.io/apimachinery/pkg/types" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" - controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/pricing" + controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/pricing" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" @@ -38,8 +38,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) var ctx context.Context @@ -55,7 +55,7 @@ func TestAWS(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) ctx, stop = context.WithCancel(ctx) @@ -98,14 +98,14 @@ var _ = Describe("Pricing", func() { ) It("should return static on-demand data if pricing API fails", func() { awsEnv.PricingAPI.NextError.Set(fmt.Errorf("failed")) - ExpectReconcileFailed(ctx, controller, types.NamespacedName{}) + _ = ExpectSingletonReconcileFailed(ctx, controller) price, ok := awsEnv.PricingProvider.OnDemandPrice("c5.large") Expect(ok).To(BeTrue()) Expect(price).To(BeNumerically(">", 0)) }) It("should return static spot data if EC2 describeSpotPriceHistory API fails", func() { awsEnv.PricingAPI.NextError.Set(fmt.Errorf("failed")) - ExpectReconcileFailed(ctx, controller, types.NamespacedName{}) + _ = ExpectSingletonReconcileFailed(ctx, controller) price, ok := awsEnv.PricingProvider.SpotPrice("c5.large", "test-zone-1a") Expect(ok).To(BeTrue()) Expect(price).To(BeNumerically(">", 0)) @@ -119,7 +119,7 @@ var _ = Describe("Pricing", func() { fake.NewOnDemandPrice("c99.large", 1.23), }, }) - ExpectReconcileFailed(ctx, controller, types.NamespacedName{}) + _ = ExpectSingletonReconcileFailed(ctx, controller) price, ok := awsEnv.PricingProvider.OnDemandPrice("c98.large") Expect(ok).To(BeTrue()) @@ -165,7 +165,7 @@ var _ = Describe("Pricing", func() { fake.NewOnDemandPrice("c99.large", 1.23), }, }) - ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, controller) price, ok := awsEnv.PricingProvider.SpotPrice("c98.large", "test-zone-1b") Expect(ok).To(BeTrue()) @@ -199,7 +199,7 @@ var _ = Describe("Pricing", func() { fake.NewOnDemandPrice("c99.large", 1.23), }, }) - ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, controller) price, ok := awsEnv.PricingProvider.SpotPrice("c98.large", "test-zone-1a") Expect(ok).To(BeTrue()) @@ -226,7 +226,7 @@ var _ = Describe("Pricing", func() { fake.NewOnDemandPrice("c99.large", 1.23), }, }) - ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, controller) _, ok := awsEnv.PricingProvider.SpotPrice("c99.large", "test-zone-1b") Expect(ok).To(BeFalse()) @@ -253,7 +253,7 @@ var _ = Describe("Pricing", func() { fake.NewOnDemandPrice("c99.large", 1.23), }, }) - ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, controller) inp := awsEnv.EC2API.DescribeSpotPriceHistoryInput.Clone() Expect(lo.Map(inp.ProductDescriptions, func(x *string, _ int) string { return *x })). To(ContainElements("Linux/UNIX", "Linux/UNIX (Amazon VPC)")) @@ -288,7 +288,7 @@ var _ = Describe("Pricing", func() { fake.NewOnDemandPrice("c5.xlarge", 1.23), }, }) - ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, controller) price, ok := awsEnv.PricingProvider.OnDemandPrice("c3.2xlarge") Expect(ok).To(BeTrue()) Expect(price).To(BeNumerically("==", 0.420000)) @@ -318,7 +318,7 @@ var _ = Describe("Pricing", func() { fake.NewOnDemandPriceWithCurrency("c99.large", 1.23, "CNY"), }, }) - ExpectReconcileSucceeded(ctx, tmpController, types.NamespacedName{}) + ExpectSingletonReconciled(ctx, tmpController) price, ok := tmpPricingProvider.OnDemandPrice("c98.large") Expect(ok).To(BeTrue()) diff --git a/pkg/fake/cloudprovider.go b/pkg/fake/cloudprovider.go index fbc9ba7ebc35..98b05ad876c3 100644 --- a/pkg/fake/cloudprovider.go +++ b/pkg/fake/cloudprovider.go @@ -17,12 +17,15 @@ package fake import ( "context" + "github.com/awslabs/operatorpkg/status" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/test" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" ) const ( @@ -37,19 +40,19 @@ type CloudProvider struct { ValidAMIs []string } -func (c *CloudProvider) Create(_ context.Context, _ *v1beta1.NodeClaim) (*v1beta1.NodeClaim, error) { +func (c *CloudProvider) Create(_ context.Context, _ *karpv1.NodeClaim) (*karpv1.NodeClaim, error) { name := test.RandomName() - return &v1beta1.NodeClaim{ + return &karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, - Status: v1beta1.NodeClaimStatus{ + Status: karpv1.NodeClaimStatus{ ProviderID: RandomProviderID(), }, }, nil } -func (c *CloudProvider) GetInstanceTypes(_ context.Context, _ *v1beta1.NodePool) ([]*corecloudprovider.InstanceType, error) { +func (c *CloudProvider) GetInstanceTypes(_ context.Context, _ *karpv1.NodePool) ([]*corecloudprovider.InstanceType, error) { if c.InstanceTypes != nil { return c.InstanceTypes, nil } @@ -58,19 +61,19 @@ func (c *CloudProvider) GetInstanceTypes(_ context.Context, _ *v1beta1.NodePool) }, nil } -func (c *CloudProvider) IsDrifted(_ context.Context, nodeClaim *v1beta1.NodeClaim) (corecloudprovider.DriftReason, error) { +func (c *CloudProvider) IsDrifted(_ context.Context, nodeClaim *karpv1.NodeClaim) (corecloudprovider.DriftReason, error) { return "drifted", nil } -func (c *CloudProvider) Get(context.Context, string) (*v1beta1.NodeClaim, error) { +func (c *CloudProvider) Get(context.Context, string) (*karpv1.NodeClaim, error) { return nil, nil } -func (c *CloudProvider) List(context.Context) ([]*v1beta1.NodeClaim, error) { +func (c *CloudProvider) List(context.Context) ([]*karpv1.NodeClaim, error) { return nil, nil } -func (c *CloudProvider) Delete(context.Context, *v1beta1.NodeClaim) error { +func (c *CloudProvider) Delete(context.Context, *karpv1.NodeClaim) error { return nil } @@ -79,6 +82,6 @@ func (c *CloudProvider) Name() string { return "fake" } -func (c *CloudProvider) GetSupportedNodeClasses() []schema.GroupVersionKind { - return []schema.GroupVersionKind{} +func (c *CloudProvider) GetSupportedNodeClasses() []status.Object { + return []status.Object{&v1.EC2NodeClass{}} } diff --git a/pkg/fake/ec2api.go b/pkg/fake/ec2api.go index ce9cf21e0087..060e0fb67134 100644 --- a/pkg/fake/ec2api.go +++ b/pkg/fake/ec2api.go @@ -31,7 +31,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/samber/lo" "k8s.io/apimachinery/pkg/util/sets" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" "sigs.k8s.io/karpenter/pkg/utils/atomic" @@ -118,7 +118,7 @@ func (e *EC2API) CreateFleetWithContext(_ context.Context, input *ec2.CreateFlee var skippedPools []CapacityPool var spotInstanceRequestID *string - if aws.StringValue(input.TargetCapacitySpecification.DefaultTargetCapacityType) == corev1beta1.CapacityTypeSpot { + if aws.StringValue(input.TargetCapacitySpecification.DefaultTargetCapacityType) == karpv1.CapacityTypeSpot { spotInstanceRequestID = aws.String(test.RandomName()) } @@ -418,6 +418,7 @@ func (e *EC2API) DescribeSubnetsWithContext(_ context.Context, input *ec2.Descri { SubnetId: aws.String("subnet-test1"), AvailabilityZone: aws.String("test-zone-1a"), + AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(100), MapPublicIpOnLaunch: aws.Bool(false), Tags: []*ec2.Tag{ @@ -428,6 +429,7 @@ func (e *EC2API) DescribeSubnetsWithContext(_ context.Context, input *ec2.Descri { SubnetId: aws.String("subnet-test2"), AvailabilityZone: aws.String("test-zone-1b"), + AvailabilityZoneId: aws.String("tstz1-1b"), AvailableIpAddressCount: aws.Int64(100), MapPublicIpOnLaunch: aws.Bool(true), Tags: []*ec2.Tag{ @@ -438,6 +440,7 @@ func (e *EC2API) DescribeSubnetsWithContext(_ context.Context, input *ec2.Descri { SubnetId: aws.String("subnet-test3"), AvailabilityZone: aws.String("test-zone-1c"), + AvailabilityZoneId: aws.String("tstz1-1c"), AvailableIpAddressCount: aws.Int64(100), Tags: []*ec2.Tag{ {Key: aws.String("Name"), Value: aws.String("test-subnet-3")}, @@ -448,6 +451,7 @@ func (e *EC2API) DescribeSubnetsWithContext(_ context.Context, input *ec2.Descri { SubnetId: aws.String("subnet-test4"), AvailabilityZone: aws.String("test-zone-1a-local"), + AvailabilityZoneId: aws.String("tstz1-1alocal"), AvailableIpAddressCount: aws.Int64(100), MapPublicIpOnLaunch: aws.Bool(true), Tags: []*ec2.Tag{ @@ -513,10 +517,10 @@ func (e *EC2API) DescribeAvailabilityZonesWithContext(context.Context, *ec2.Desc return e.DescribeAvailabilityZonesOutput.Clone(), nil } return &ec2.DescribeAvailabilityZonesOutput{AvailabilityZones: []*ec2.AvailabilityZone{ - {ZoneName: aws.String("test-zone-1a"), ZoneId: aws.String("testzone1a"), ZoneType: aws.String("availability-zone")}, - {ZoneName: aws.String("test-zone-1b"), ZoneId: aws.String("testzone1b"), ZoneType: aws.String("availability-zone")}, - {ZoneName: aws.String("test-zone-1c"), ZoneId: aws.String("testzone1c"), ZoneType: aws.String("availability-zone")}, - {ZoneName: aws.String("test-zone-1a-local"), ZoneId: aws.String("testzone1alocal"), ZoneType: aws.String("local-zone")}, + {ZoneName: aws.String("test-zone-1a"), ZoneId: aws.String("tstz1-1a"), ZoneType: aws.String("availability-zone")}, + {ZoneName: aws.String("test-zone-1b"), ZoneId: aws.String("tstz1-1b"), ZoneType: aws.String("availability-zone")}, + {ZoneName: aws.String("test-zone-1c"), ZoneId: aws.String("tstz1-1c"), ZoneType: aws.String("availability-zone")}, + {ZoneName: aws.String("test-zone-1a-local"), ZoneId: aws.String("tstz1-1alocal"), ZoneType: aws.String("local-zone")}, }}, nil } @@ -610,6 +614,14 @@ func (e *EC2API) DescribeInstanceTypeOfferingsWithContext(_ context.Context, _ * InstanceType: aws.String("g4dn.8xlarge"), Location: aws.String("test-zone-1b"), }, + { + InstanceType: aws.String("g4ad.16xlarge"), + Location: aws.String("test-zone-1a"), + }, + { + InstanceType: aws.String("g4ad.16xlarge"), + Location: aws.String("test-zone-1b"), + }, { InstanceType: aws.String("t3.large"), Location: aws.String("test-zone-1a"), diff --git a/pkg/fake/iamapi.go b/pkg/fake/iamapi.go index 620c514da162..54e377b74532 100644 --- a/pkg/fake/iamapi.go +++ b/pkg/fake/iamapi.go @@ -37,6 +37,7 @@ type IAMAPIBehavior struct { CreateInstanceProfileBehavior MockedFunction[iam.CreateInstanceProfileInput, iam.CreateInstanceProfileOutput] DeleteInstanceProfileBehavior MockedFunction[iam.DeleteInstanceProfileInput, iam.DeleteInstanceProfileOutput] AddRoleToInstanceProfileBehavior MockedFunction[iam.AddRoleToInstanceProfileInput, iam.AddRoleToInstanceProfileOutput] + TagInstanceProfileBehavior MockedFunction[iam.TagInstanceProfileInput, iam.TagInstanceProfileOutput] RemoveRoleFromInstanceProfileBehavior MockedFunction[iam.RemoveRoleFromInstanceProfileInput, iam.RemoveRoleFromInstanceProfileOutput] } @@ -112,6 +113,21 @@ func (s *IAMAPI) DeleteInstanceProfileWithContext(_ context.Context, input *iam. }) } +func (s *IAMAPI) TagInstanceProfileWithContext(_ context.Context, input *iam.TagInstanceProfileInput, _ ...request.Option) (*iam.TagInstanceProfileOutput, error) { + return s.TagInstanceProfileBehavior.Invoke(input, func(output *iam.TagInstanceProfileInput) (*iam.TagInstanceProfileOutput, error) { + s.Lock() + defer s.Unlock() + + if profile, ok := s.InstanceProfiles[aws.StringValue(input.InstanceProfileName)]; ok { + profile.Tags = lo.UniqBy(append(input.Tags, profile.Tags...), func(t *iam.Tag) string { + return lo.FromPtr(t.Key) + }) + return nil, nil + } + return nil, awserr.New(iam.ErrCodeNoSuchEntityException, fmt.Sprintf("Instance Profile %s cannot be found", aws.StringValue(input.InstanceProfileName)), nil) + }) +} + func (s *IAMAPI) AddRoleToInstanceProfileWithContext(_ context.Context, input *iam.AddRoleToInstanceProfileInput, _ ...request.Option) (*iam.AddRoleToInstanceProfileOutput, error) { return s.AddRoleToInstanceProfileBehavior.Invoke(input, func(output *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error) { s.Lock() diff --git a/pkg/fake/ssmapi.go b/pkg/fake/ssmapi.go index b9d0e1711d84..c772d55d842c 100644 --- a/pkg/fake/ssmapi.go +++ b/pkg/fake/ssmapi.go @@ -18,10 +18,9 @@ import ( "context" "fmt" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/mitchellh/hashstructure/v2" + "github.com/Pallinder/go-randomdata" + "github.com/samber/lo" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/ssm" "github.com/aws/aws-sdk-go/service/ssm/ssmiface" @@ -32,36 +31,54 @@ type SSMAPI struct { Parameters map[string]string GetParameterOutput *ssm.GetParameterOutput WantErr error + + defaultParameters map[string]string } func NewSSMAPI() *SSMAPI { - return &SSMAPI{} + return &SSMAPI{ + defaultParameters: map[string]string{}, + } } func (a SSMAPI) GetParameterWithContext(_ context.Context, input *ssm.GetParameterInput, _ ...request.Option) (*ssm.GetParameterOutput, error) { + parameter := lo.FromPtr(input.Name) if a.WantErr != nil { - return nil, a.WantErr + return &ssm.GetParameterOutput{}, a.WantErr } - if len(a.Parameters) > 0 { - if amiID, ok := a.Parameters[*input.Name]; ok { - return &ssm.GetParameterOutput{ - Parameter: &ssm.Parameter{Value: aws.String(amiID)}, - }, nil - } - return nil, awserr.New(ssm.ErrCodeParameterNotFound, fmt.Sprintf("%s couldn't be found", *input.Name), nil) - } - hc, _ := hashstructure.Hash(input.Name, hashstructure.FormatV2, nil) if a.GetParameterOutput != nil { return a.GetParameterOutput, nil } + if len(a.Parameters) != 0 { + value, ok := a.Parameters[parameter] + if !ok { + return &ssm.GetParameterOutput{}, fmt.Errorf("parameter %q not found", lo.FromPtr(input.Name)) + } + return &ssm.GetParameterOutput{ + Parameter: &ssm.Parameter{ + Name: lo.ToPtr(parameter), + Value: lo.ToPtr(value), + }, + }, nil + } + // Cache default parameters that was successive calls for the same parameter return the same result + value, ok := a.defaultParameters[parameter] + if !ok { + value = fmt.Sprintf("ami-%s", randomdata.Alphanumeric(16)) + a.defaultParameters[parameter] = value + } return &ssm.GetParameterOutput{ - Parameter: &ssm.Parameter{Value: aws.String(fmt.Sprintf("test-ami-id-%x", hc))}, + Parameter: &ssm.Parameter{ + Name: lo.ToPtr(parameter), + Value: lo.ToPtr(value), + }, }, nil } func (a *SSMAPI) Reset() { - a.GetParameterOutput = nil a.Parameters = nil + a.GetParameterOutput = nil a.WantErr = nil + a.defaultParameters = map[string]string{} } diff --git a/pkg/fake/zz_generated.describe_instance_types.go b/pkg/fake/zz_generated.describe_instance_types.go index ad2b7da73e67..da2762eee5f6 100644 --- a/pkg/fake/zz_generated.describe_instance_types.go +++ b/pkg/fake/zz_generated.describe_instance_types.go @@ -45,6 +45,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(4096), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(630), + BaselineIops: aws.Int64(3600), + BaselineThroughputInMBps: aws.Float64(78.75), + MaximumBandwidthInMbps: aws.Int64(4750), + MaximumIops: aws.Int64(20000), + MaximumThroughputInMBps: aws.Float64(593.75), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, NetworkInfo: &ec2.NetworkInfo{ MaximumNetworkInterfaces: aws.Int64(3), Ipv4AddressesPerInterface: aws.Int64(10), @@ -76,6 +89,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(786432), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(19000), + BaselineIops: aws.Int64(80000), + BaselineThroughputInMBps: aws.Float64(2375.00), + MaximumBandwidthInMbps: aws.Int64(19000), + MaximumIops: aws.Int64(80000), + MaximumThroughputInMBps: aws.Float64(2375.00), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, GpuInfo: &ec2.GpuInfo{ Gpus: []*ec2.GpuDeviceInfo{ { @@ -119,6 +145,65 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ }, }, }, + { + InstanceType: aws.String("g4ad.16xlarge"), + SupportedUsageClasses: aws.StringSlice([]string{"on-demand", "spot"}), + SupportedVirtualizationTypes: aws.StringSlice([]string{"hvm"}), + BurstablePerformanceSupported: aws.Bool(false), + BareMetal: aws.Bool(false), + Hypervisor: aws.String("nitro"), + ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("AMD"), + SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), + }, + VCpuInfo: &ec2.VCpuInfo{ + DefaultCores: aws.Int64(32), + DefaultVCpus: aws.Int64(64), + }, + MemoryInfo: &ec2.MemoryInfo{ + SizeInMiB: aws.Int64(262144), + }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(6300), + BaselineIops: aws.Int64(26667), + BaselineThroughputInMBps: aws.Float64(787.50), + MaximumBandwidthInMbps: aws.Int64(6300), + MaximumIops: aws.Int64(26667), + MaximumThroughputInMBps: aws.Float64(787.50), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, + GpuInfo: &ec2.GpuInfo{ + Gpus: []*ec2.GpuDeviceInfo{ + { + Name: aws.String("Radeon Pro V520"), + Manufacturer: aws.String("AMD"), + Count: aws.Int64(4), + MemoryInfo: &ec2.GpuDeviceMemoryInfo{ + SizeInMiB: aws.Int64(8192), + }, + }, + }, + }, + InstanceStorageInfo: &ec2.InstanceStorageInfo{NvmeSupport: aws.String("required"), + TotalSizeInGB: aws.Int64(2400), + }, + NetworkInfo: &ec2.NetworkInfo{ + MaximumNetworkInterfaces: aws.Int64(8), + Ipv4AddressesPerInterface: aws.Int64(30), + EncryptionInTransitSupported: aws.Bool(true), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{ + { + NetworkCardIndex: aws.Int64(0), + MaximumNetworkInterfaces: aws.Int64(8), + }, + }, + }, + }, { InstanceType: aws.String("g4dn.8xlarge"), SupportedUsageClasses: aws.StringSlice([]string{"on-demand", "spot"}), @@ -137,6 +222,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(131072), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(9500), + BaselineIops: aws.Int64(40000), + BaselineThroughputInMBps: aws.Float64(1187.50), + MaximumBandwidthInMbps: aws.Int64(9500), + MaximumIops: aws.Int64(40000), + MaximumThroughputInMBps: aws.Float64(1187.50), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, GpuInfo: &ec2.GpuInfo{ Gpus: []*ec2.GpuDeviceInfo{ { @@ -186,6 +284,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(16384), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(1190), + BaselineIops: aws.Int64(6000), + BaselineThroughputInMBps: aws.Float64(148.75), + MaximumBandwidthInMbps: aws.Int64(4750), + MaximumIops: aws.Int64(20000), + MaximumThroughputInMBps: aws.Float64(593.75), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, InferenceAcceleratorInfo: &ec2.InferenceAcceleratorInfo{ Accelerators: []*ec2.InferenceDeviceInfo{ { @@ -226,6 +337,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(49152), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(4750), + BaselineIops: aws.Int64(20000), + BaselineThroughputInMBps: aws.Float64(593.75), + MaximumBandwidthInMbps: aws.Int64(4750), + MaximumIops: aws.Int64(20000), + MaximumThroughputInMBps: aws.Float64(593.75), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, InferenceAcceleratorInfo: &ec2.InferenceAcceleratorInfo{ Accelerators: []*ec2.InferenceDeviceInfo{ { @@ -266,6 +390,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(8192), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(650), + BaselineIops: aws.Int64(3600), + BaselineThroughputInMBps: aws.Float64(81.25), + MaximumBandwidthInMbps: aws.Int64(4750), + MaximumIops: aws.Int64(18750), + MaximumThroughputInMBps: aws.Float64(593.75), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, NetworkInfo: &ec2.NetworkInfo{ MaximumNetworkInterfaces: aws.Int64(3), Ipv4AddressesPerInterface: aws.Int64(10), @@ -297,6 +434,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(393216), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(19000), + BaselineIops: aws.Int64(80000), + BaselineThroughputInMBps: aws.Float64(2375.00), + MaximumBandwidthInMbps: aws.Int64(19000), + MaximumIops: aws.Int64(80000), + MaximumThroughputInMBps: aws.Float64(2375.00), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, NetworkInfo: &ec2.NetworkInfo{ MaximumNetworkInterfaces: aws.Int64(15), Ipv4AddressesPerInterface: aws.Int64(50), @@ -328,6 +478,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(16384), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(1150), + BaselineIops: aws.Int64(6000), + BaselineThroughputInMBps: aws.Float64(143.75), + MaximumBandwidthInMbps: aws.Int64(4750), + MaximumIops: aws.Int64(18750), + MaximumThroughputInMBps: aws.Float64(593.75), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, NetworkInfo: &ec2.NetworkInfo{ MaximumNetworkInterfaces: aws.Int64(4), Ipv4AddressesPerInterface: aws.Int64(15), @@ -359,6 +522,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(524288), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(100000), + BaselineIops: aws.Int64(400000), + BaselineThroughputInMBps: aws.Float64(12500.00), + MaximumBandwidthInMbps: aws.Int64(100000), + MaximumIops: aws.Int64(400000), + MaximumThroughputInMBps: aws.Float64(12500.00), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, InstanceStorageInfo: &ec2.InstanceStorageInfo{NvmeSupport: aws.String("required"), TotalSizeInGB: aws.Int64(7600), }, @@ -366,18 +542,18 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ EfaInfo: &ec2.EfaInfo{ MaximumEfaInterfaces: aws.Int64(2), }, - MaximumNetworkInterfaces: aws.Int64(14), + MaximumNetworkInterfaces: aws.Int64(16), Ipv4AddressesPerInterface: aws.Int64(50), EncryptionInTransitSupported: aws.Bool(true), DefaultNetworkCardIndex: aws.Int64(0), NetworkCards: []*ec2.NetworkCardInfo{ { NetworkCardIndex: aws.Int64(0), - MaximumNetworkInterfaces: aws.Int64(7), + MaximumNetworkInterfaces: aws.Int64(8), }, { NetworkCardIndex: aws.Int64(1), - MaximumNetworkInterfaces: aws.Int64(7), + MaximumNetworkInterfaces: aws.Int64(8), }, }, }, @@ -400,6 +576,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(249856), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(7000), + BaselineIops: aws.Int64(40000), + BaselineThroughputInMBps: aws.Float64(875.00), + MaximumBandwidthInMbps: aws.Int64(7000), + MaximumIops: aws.Int64(40000), + MaximumThroughputInMBps: aws.Float64(875.00), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("unsupported"), + }, GpuInfo: &ec2.GpuInfo{ Gpus: []*ec2.GpuDeviceInfo{ { @@ -443,6 +632,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(8192), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(695), + BaselineIops: aws.Int64(4000), + BaselineThroughputInMBps: aws.Float64(86.88), + MaximumBandwidthInMbps: aws.Int64(2780), + MaximumIops: aws.Int64(15700), + MaximumThroughputInMBps: aws.Float64(347.50), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, NetworkInfo: &ec2.NetworkInfo{ MaximumNetworkInterfaces: aws.Int64(3), Ipv4AddressesPerInterface: aws.Int64(12), @@ -474,6 +676,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(4096), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(347), + BaselineIops: aws.Int64(2000), + BaselineThroughputInMBps: aws.Float64(43.38), + MaximumBandwidthInMbps: aws.Int64(2085), + MaximumIops: aws.Int64(11800), + MaximumThroughputInMBps: aws.Float64(260.62), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, NetworkInfo: &ec2.NetworkInfo{ MaximumNetworkInterfaces: aws.Int64(3), Ipv4AddressesPerInterface: aws.Int64(6), @@ -505,6 +720,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(2048), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(174), + BaselineIops: aws.Int64(1000), + BaselineThroughputInMBps: aws.Float64(21.75), + MaximumBandwidthInMbps: aws.Int64(2085), + MaximumIops: aws.Int64(11800), + MaximumThroughputInMBps: aws.Float64(260.62), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, NetworkInfo: &ec2.NetworkInfo{ MaximumNetworkInterfaces: aws.Int64(3), Ipv4AddressesPerInterface: aws.Int64(4), @@ -536,6 +764,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(16384), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(695), + BaselineIops: aws.Int64(4000), + BaselineThroughputInMBps: aws.Float64(86.88), + MaximumBandwidthInMbps: aws.Int64(2780), + MaximumIops: aws.Int64(15700), + MaximumThroughputInMBps: aws.Float64(347.50), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, NetworkInfo: &ec2.NetworkInfo{ MaximumNetworkInterfaces: aws.Int64(4), Ipv4AddressesPerInterface: aws.Int64(15), @@ -567,6 +808,19 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MemoryInfo: &ec2.MemoryInfo{ SizeInMiB: aws.Int64(32768), }, + EbsInfo: &ec2.EbsInfo{ + EbsOptimizedInfo: &ec2.EbsOptimizedInfo{ + BaselineBandwidthInMbps: aws.Int64(5000), + BaselineIops: aws.Int64(16250), + BaselineThroughputInMBps: aws.Float64(625.00), + MaximumBandwidthInMbps: aws.Int64(20000), + MaximumIops: aws.Int64(65000), + MaximumThroughputInMBps: aws.Float64(2500.00), + }, + EbsOptimizedSupport: aws.String("default"), + EncryptionSupport: aws.String("supported"), + NvmeSupport: aws.String("required"), + }, InstanceStorageInfo: &ec2.InstanceStorageInfo{NvmeSupport: aws.String("required"), TotalSizeInGB: aws.Int64(474), }, diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 39be55788322..fd7683fe8f4c 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -20,12 +20,11 @@ import ( "errors" "fmt" "net" - "time" + "os" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" awsclient "github.com/aws/aws-sdk-go/aws/client" - "github.com/aws/aws-sdk-go/aws/credentials/stscreds" "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/request" @@ -36,6 +35,7 @@ import ( "github.com/aws/aws-sdk-go/service/eks/eksiface" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/ssm" + prometheusv1 "github.com/jonathan-innis/aws-sdk-go-prometheus/v1" "github.com/patrickmn/go-cache" "github.com/samber/lo" corev1 "k8s.io/api/core/v1" @@ -43,14 +43,13 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/transport" - "knative.dev/pkg/logging" - "knative.dev/pkg/ptr" + "sigs.k8s.io/controller-runtime/pkg/log" + crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/operator" - "sigs.k8s.io/karpenter/pkg/operator/scheme" - "github.com/aws/karpenter-provider-aws/pkg/apis" awscache "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" @@ -60,13 +59,14 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" "github.com/aws/karpenter-provider-aws/pkg/providers/securitygroup" + ssmp "github.com/aws/karpenter-provider-aws/pkg/providers/ssm" "github.com/aws/karpenter-provider-aws/pkg/providers/subnet" "github.com/aws/karpenter-provider-aws/pkg/providers/version" ) func init() { - lo.Must0(apis.AddToScheme(scheme.Scheme)) - corev1beta1.NormalizedLabels = lo.Assign(corev1beta1.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": corev1.LabelTopologyZone}) + karpv1beta1.NormalizedLabels = lo.Assign(karpv1.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": corev1.LabelTopologyZone}) + karpv1.NormalizedLabels = lo.Assign(karpv1.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": corev1.LabelTopologyZone}) } // Operator is injected into the AWS CloudProvider's factories @@ -86,6 +86,7 @@ type Operator struct { VersionProvider version.Provider InstanceTypesProvider instancetype.Provider InstanceProvider instance.Provider + SSMProvider ssmp.Provider } func NewOperator(ctx context.Context, operator *operator.Operator) (context.Context, *Operator) { @@ -93,46 +94,46 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont STSRegionalEndpoint: endpoints.RegionalSTSEndpoint, } - if assumeRoleARN := options.FromContext(ctx).AssumeRoleARN; assumeRoleARN != "" { - config.Credentials = stscreds.NewCredentials(session.Must(session.NewSession()), assumeRoleARN, - func(provider *stscreds.AssumeRoleProvider) { SetDurationAndExpiry(ctx, provider) }) - } - - sess := WithUserAgent(session.Must(session.NewSession( + // prometheusv1.WithPrometheusMetrics is used until the upstream aws-sdk-go or aws-sdk-go-v2 supports + // Prometheus metrics for client-side metrics out-of-the-box + // See: https://github.com/aws/aws-sdk-go-v2/issues/1744 + sess := prometheusv1.WithPrometheusMetrics(WithUserAgent(session.Must(session.NewSession( request.WithRetryer( config, awsclient.DefaultRetryer{NumMaxRetries: awsclient.DefaultRetryerMaxNumRetries}, ), - ))) + ))), crmetrics.Registry) if *sess.Config.Region == "" { - logging.FromContext(ctx).Debug("retrieving region from IMDS") + log.FromContext(ctx).V(1).Info("retrieving region from IMDS") region, err := ec2metadata.New(sess).Region() *sess.Config.Region = lo.Must(region, err, "failed to get region from metadata server") } ec2api := ec2.New(sess) if err := CheckEC2Connectivity(ctx, ec2api); err != nil { - logging.FromContext(ctx).Fatalf("Checking EC2 API connectivity, %s", err) + log.FromContext(ctx).Error(err, "ec2 api connectivity check failed") + os.Exit(1) } - logging.FromContext(ctx).With("region", *sess.Config.Region).Debugf("discovered region") + log.FromContext(ctx).WithValues("region", *sess.Config.Region).V(1).Info("discovered region") clusterEndpoint, err := ResolveClusterEndpoint(ctx, eks.New(sess)) if err != nil { - logging.FromContext(ctx).Fatalf("unable to detect the cluster endpoint, %s", err) + log.FromContext(ctx).Error(err, "failed detecting cluster endpoint") + os.Exit(1) } else { - logging.FromContext(ctx).With("cluster-endpoint", clusterEndpoint).Debugf("discovered cluster endpoint") + log.FromContext(ctx).WithValues("cluster-endpoint", clusterEndpoint).V(1).Info("discovered cluster endpoint") } // We perform best-effort on resolving the kube-dns IP kubeDNSIP, err := KubeDNSIP(ctx, operator.KubernetesInterface) if err != nil { // If we fail to get the kube-dns IP, we don't want to crash because this causes issues with custom DNS setups // https://github.com/aws/karpenter-provider-aws/issues/2787 - logging.FromContext(ctx).Debugf("unable to detect the IP of the kube-dns service, %s", err) + log.FromContext(ctx).V(1).Info(fmt.Sprintf("unable to detect the IP of the kube-dns service, %s", err)) } else { - logging.FromContext(ctx).With("kube-dns-ip", kubeDNSIP).Debugf("discovered kube dns") + log.FromContext(ctx).WithValues("kube-dns-ip", kubeDNSIP).V(1).Info("discovered kube dns") } unavailableOfferingsCache := awscache.NewUnavailableOfferings() - subnetProvider := subnet.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) + subnetProvider := subnet.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), cache.New(awscache.AvailableIPAddressTTL, awscache.DefaultCleanupInterval), cache.New(awscache.AssociatePublicIPAddressTTL, awscache.DefaultCleanupInterval)) securityGroupProvider := securitygroup.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) instanceProfileProvider := instanceprofile.NewDefaultProvider(*sess.Config.Region, iam.New(sess), cache.New(awscache.InstanceProfileTTL, awscache.DefaultCleanupInterval)) pricingProvider := pricing.NewDefaultProvider( @@ -142,7 +143,8 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont *sess.Config.Region, ) versionProvider := version.NewDefaultProvider(operator.KubernetesInterface, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) - amiProvider := amifamily.NewDefaultProvider(versionProvider, ssm.New(sess), ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) + ssmProvider := ssmp.NewDefaultProvider(ssm.New(sess), cache.New(awscache.SSMGetParametersByPathTTL, awscache.DefaultCleanupInterval)) + amiProvider := amifamily.NewDefaultProvider(versionProvider, ssmProvider, ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) amiResolver := amifamily.NewResolver(amiProvider) launchTemplateProvider := launchtemplate.NewDefaultProvider( ctx, @@ -190,6 +192,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont PricingProvider: pricingProvider, InstanceTypesProvider: instanceTypeProvider, InstanceProvider: instanceProvider, + SSMProvider: ssmProvider, } } @@ -241,7 +244,7 @@ func GetCABundle(ctx context.Context, restConfig *rest.Config) (*string, error) if err != nil { return nil, fmt.Errorf("discovering caBundle, loading TLS config, %w", err) } - return ptr.String(base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData)), nil + return lo.ToPtr(base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData)), nil } func KubeDNSIP(ctx context.Context, kubernetesInterface kubernetes.Interface) (net.IP, error) { @@ -258,8 +261,3 @@ func KubeDNSIP(ctx context.Context, kubernetesInterface kubernetes.Interface) (n } return kubeDNSIP, nil } - -func SetDurationAndExpiry(ctx context.Context, provider *stscreds.AssumeRoleProvider) { - provider.Duration = options.FromContext(ctx).AssumeRoleDuration - provider.ExpiryWindow = time.Duration(10) * time.Second -} diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 84d9c38c5268..4a5ef2c0f7d5 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -20,10 +20,11 @@ import ( "flag" "fmt" "os" - "time" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/utils/env" + + "github.com/aws/karpenter-provider-aws/pkg/utils" ) func init() { @@ -33,8 +34,6 @@ func init() { type optionsKey struct{} type Options struct { - AssumeRoleARN string - AssumeRoleDuration time.Duration ClusterCABundle string ClusterName string ClusterEndpoint string @@ -45,14 +44,12 @@ type Options struct { } func (o *Options) AddFlags(fs *coreoptions.FlagSet) { - fs.StringVar(&o.AssumeRoleARN, "assume-role-arn", env.WithDefaultString("ASSUME_ROLE_ARN", ""), "Role to assume for calling AWS services.") - fs.DurationVar(&o.AssumeRoleDuration, "assume-role-duration", env.WithDefaultDuration("ASSUME_ROLE_DURATION", 15*time.Minute), "Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set.") fs.StringVar(&o.ClusterCABundle, "cluster-ca-bundle", env.WithDefaultString("CLUSTER_CA_BUNDLE", ""), "Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.") fs.StringVar(&o.ClusterName, "cluster-name", env.WithDefaultString("CLUSTER_NAME", ""), "[REQUIRED] The kubernetes cluster name for resource discovery.") fs.StringVar(&o.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.") fs.BoolVarWithEnv(&o.IsolatedVPC, "isolated-vpc", "ISOLATED_VPC", false, "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS on-demand pricing endpoint.") - fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") - fs.StringVar(&o.InterruptionQueue, "interruption-queue", env.WithDefaultString("INTERRUPTION_QUEUE", ""), "Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") + fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", utils.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") + fs.StringVar(&o.InterruptionQueue, "interruption-queue", env.WithDefaultString("INTERRUPTION_QUEUE", ""), "Interruption queue is the name of the SQS queue used for processing interruption events from EC2. Interruption handling is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") fs.IntVar(&o.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.") } diff --git a/pkg/operator/options/options_validation.go b/pkg/operator/options/options_validation.go index 1c4a733700ec..b866648fa9b2 100644 --- a/pkg/operator/options/options_validation.go +++ b/pkg/operator/options/options_validation.go @@ -17,7 +17,6 @@ package options import ( "fmt" "net/url" - "time" "go.uber.org/multierr" ) @@ -26,19 +25,11 @@ func (o Options) Validate() error { return multierr.Combine( o.validateEndpoint(), o.validateVMMemoryOverheadPercent(), - o.validateAssumeRoleDuration(), o.validateReservedENIs(), o.validateRequiredFields(), ) } -func (o Options) validateAssumeRoleDuration() error { - if o.AssumeRoleDuration < time.Minute*15 { - return fmt.Errorf("assume-role-duration cannot be less than 15 minutes") - } - return nil -} - func (o Options) validateEndpoint() error { if o.ClusterEndpoint == "" { return nil diff --git a/pkg/operator/options/suite_test.go b/pkg/operator/options/suite_test.go index 6b7e39715149..14e60dd0332c 100644 --- a/pkg/operator/options/suite_test.go +++ b/pkg/operator/options/suite_test.go @@ -19,7 +19,6 @@ import ( "flag" "os" "testing" - "time" "github.com/samber/lo" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" @@ -29,7 +28,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) var ctx context.Context @@ -57,8 +56,6 @@ var _ = Describe("Options", func() { It("should correctly override default vars when CLI flags are set", func() { opts.AddFlags(fs) err := opts.Parse(fs, - "--assume-role-arn", "env-role", - "--assume-role-duration", "20m", "--cluster-ca-bundle", "env-bundle", "--cluster-name", "env-cluster", "--cluster-endpoint", "https://env-cluster", @@ -68,8 +65,6 @@ var _ = Describe("Options", func() { "--reserved-enis", "10") Expect(err).ToNot(HaveOccurred()) expectOptionsEqual(opts, test.Options(test.OptionsFields{ - AssumeRoleARN: lo.ToPtr("env-role"), - AssumeRoleDuration: lo.ToPtr(20 * time.Minute), ClusterCABundle: lo.ToPtr("env-bundle"), ClusterName: lo.ToPtr("env-cluster"), ClusterEndpoint: lo.ToPtr("https://env-cluster"), @@ -80,8 +75,6 @@ var _ = Describe("Options", func() { })) }) It("should correctly fallback to env vars when CLI flags aren't set", func() { - os.Setenv("ASSUME_ROLE_ARN", "env-role") - os.Setenv("ASSUME_ROLE_DURATION", "20m") os.Setenv("CLUSTER_CA_BUNDLE", "env-bundle") os.Setenv("CLUSTER_NAME", "env-cluster") os.Setenv("CLUSTER_ENDPOINT", "https://env-cluster") @@ -96,8 +89,6 @@ var _ = Describe("Options", func() { err := opts.Parse(fs) Expect(err).ToNot(HaveOccurred()) expectOptionsEqual(opts, test.Options(test.OptionsFields{ - AssumeRoleARN: lo.ToPtr("env-role"), - AssumeRoleDuration: lo.ToPtr(20 * time.Minute), ClusterCABundle: lo.ToPtr("env-bundle"), ClusterName: lo.ToPtr("env-cluster"), ClusterEndpoint: lo.ToPtr("https://env-cluster"), @@ -116,10 +107,6 @@ var _ = Describe("Options", func() { err := opts.Parse(fs) Expect(err).To(HaveOccurred()) }) - It("should fail when assume role duration is less than 15 minutes", func() { - err := opts.Parse(fs, "--cluster-name", "test-cluster", "--assume-role-duration", "1s") - Expect(err).To(HaveOccurred()) - }) It("should fail when clusterEndpoint is invalid (not absolute)", func() { err := opts.Parse(fs, "--cluster-name", "test-cluster", "--cluster-endpoint", "00000000000000000000000.gr7.us-west-2.eks.amazonaws.com") Expect(err).To(HaveOccurred()) @@ -137,8 +124,6 @@ var _ = Describe("Options", func() { func expectOptionsEqual(optsA *options.Options, optsB *options.Options) { GinkgoHelper() - Expect(optsA.AssumeRoleARN).To(Equal(optsB.AssumeRoleARN)) - Expect(optsA.AssumeRoleDuration).To(Equal(optsB.AssumeRoleDuration)) Expect(optsA.ClusterCABundle).To(Equal(optsB.ClusterCABundle)) Expect(optsA.ClusterName).To(Equal(optsB.ClusterName)) Expect(optsA.ClusterEndpoint).To(Equal(optsB.ClusterEndpoint)) diff --git a/pkg/operator/suite_test.go b/pkg/operator/suite_test.go index f725b2f23652..13aed00a0fd2 100644 --- a/pkg/operator/suite_test.go +++ b/pkg/operator/suite_test.go @@ -19,10 +19,11 @@ import ( "errors" "testing" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + "github.com/aws/aws-sdk-go/service/eks" "github.com/samber/lo" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" @@ -33,8 +34,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) var ctx context.Context @@ -49,7 +50,7 @@ func TestAWS(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) ctx, stop = context.WithCancel(ctx) fakeEKSAPI = &fake.EKSAPI{} diff --git a/pkg/providers/amifamily/al2.go b/pkg/providers/amifamily/al2.go index ad71153d0a3c..ebd385250dc7 100644 --- a/pkg/providers/amifamily/al2.go +++ b/pkg/providers/amifamily/al2.go @@ -15,19 +15,23 @@ limitations under the License. package amifamily import ( + "context" "fmt" "github.com/aws/aws-sdk-go/aws" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/samber/lo" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/scheduling" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap" + "github.com/aws/karpenter-provider-aws/pkg/providers/ssm" ) type AL2 struct { @@ -35,47 +39,52 @@ type AL2 struct { *Options } -// DefaultAMIs returns the AMI name, and Requirements, with an SSM query -func (a AL2) DefaultAMIs(version string) []DefaultAMIOutput { - return []DefaultAMIOutput{ - { - Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version), - Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), - scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist), - ), - }, - { - Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version), - Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), - scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpExists), - ), - }, - { - Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version), - Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), - scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpExists), - ), - }, - { - Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-%s/recommended/image_id", version, corev1beta1.ArchitectureArm64), - Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), - scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist), - ), - }, +func (a AL2) DescribeImageQuery(ctx context.Context, ssmProvider ssm.Provider, k8sVersion string, amiVersion string) (DescribeImageQuery, error) { + ids := map[string][]Variant{} + for path, variants := range map[string][]Variant{ + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/%s/image_id", k8sVersion, lo.Ternary( + amiVersion == AMIVersionLatest, + "recommended", + fmt.Sprintf("amazon-eks-node-%s-%s", k8sVersion, amiVersion), + )): {VariantStandard}, + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/%s/image_id", k8sVersion, lo.Ternary( + amiVersion == AMIVersionLatest, + "recommended", + fmt.Sprintf("amazon-eks-arm64-node-%s-%s", k8sVersion, amiVersion), + )): {VariantStandard}, + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/%s/image_id", k8sVersion, lo.Ternary( + amiVersion == AMIVersionLatest, + "recommended", + fmt.Sprintf("amazon-eks-gpu-node-%s-%s", k8sVersion, amiVersion), + )): {VariantNeuron, VariantNvidia}, + } { + imageID, err := ssmProvider.Get(ctx, path) + if err != nil { + continue + } + ids[imageID] = variants + } + // Failed to discover any AMIs, we should short circuit AMI discovery + if len(ids) == 0 { + return DescribeImageQuery{}, fmt.Errorf(`failed to discover any AMIs for alias "al2@%s"`, amiVersion) } + + return DescribeImageQuery{ + Filters: []*ec2.Filter{{ + Name: lo.ToPtr("image-id"), + Values: lo.ToSlicePtr(lo.Keys(ids)), + }}, + KnownRequirements: lo.MapValues(ids, func(variants []Variant, _ string) []scheduling.Requirements { + return lo.Map(variants, func(v Variant, _ int) scheduling.Requirements { return v.Requirements() }) + }), + }, nil } // UserData returns the exact same string for equivalent input, // even if elements of those inputs are in differing orders, // guaranteeing it won't cause spurious hash differences. // AL2 userdata also works on Ubuntu -func (a AL2) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ []*cloudprovider.InstanceType, customUserData *string, instanceStorePolicy *v1beta1.InstanceStorePolicy) bootstrap.Bootstrapper { +func (a AL2) UserData(kubeletConfig *v1.KubeletConfiguration, taints []corev1.Taint, labels map[string]string, caBundle *string, _ []*cloudprovider.InstanceType, customUserData *string, instanceStorePolicy *v1.InstanceStorePolicy) bootstrap.Bootstrapper { return bootstrap.EKS{ Options: bootstrap.Options{ ClusterName: a.Options.ClusterName, @@ -91,8 +100,8 @@ func (a AL2) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints [] } // DefaultBlockDeviceMappings returns the default block device mappings for the AMI Family -func (a AL2) DefaultBlockDeviceMappings() []*v1beta1.BlockDeviceMapping { - return []*v1beta1.BlockDeviceMapping{{ +func (a AL2) DefaultBlockDeviceMappings() []*v1.BlockDeviceMapping { + return []*v1.BlockDeviceMapping{{ DeviceName: a.EphemeralBlockDevice(), EBS: &DefaultEBS, }} diff --git a/pkg/providers/amifamily/al2023.go b/pkg/providers/amifamily/al2023.go index d586d66e35fd..bde9161b0be9 100644 --- a/pkg/providers/amifamily/al2023.go +++ b/pkg/providers/amifamily/al2023.go @@ -15,16 +15,19 @@ limitations under the License. package amifamily import ( + "context" "fmt" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/scheduling" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + "github.com/aws/aws-sdk-go/service/ec2" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap" + "github.com/aws/karpenter-provider-aws/pkg/providers/ssm" ) type AL2023 struct { @@ -32,24 +35,47 @@ type AL2023 struct { *Options } -func (a AL2023) DefaultAMIs(version string) []DefaultAMIOutput { - return []DefaultAMIOutput{ - { - Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", version), - Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), - ), - }, - { - Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version), - Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), - ), - }, +func (a AL2023) DescribeImageQuery(ctx context.Context, ssmProvider ssm.Provider, k8sVersion string, amiVersion string) (DescribeImageQuery, error) { + ids := map[string]Variant{} + for arch, variants := range map[string][]Variant{ + "x86_64": []Variant{VariantStandard, VariantNvidia, VariantNeuron}, + "arm64": []Variant{VariantStandard}, + } { + for _, variant := range variants { + path := a.resolvePath(arch, string(variant), k8sVersion, amiVersion) + imageID, err := ssmProvider.Get(ctx, path) + if err != nil { + continue + } + ids[imageID] = variant + } + } + // Failed to discover any AMIs, we should short circuit AMI discovery + if len(ids) == 0 { + return DescribeImageQuery{}, fmt.Errorf(`failed to discover AMIs for alias "al2023@%s"`, amiVersion) } + + return DescribeImageQuery{ + Filters: []*ec2.Filter{{ + Name: lo.ToPtr("image-id"), + Values: lo.ToSlicePtr(lo.Keys(ids)), + }}, + KnownRequirements: lo.MapValues(ids, func(v Variant, _ string) []scheduling.Requirements { + return []scheduling.Requirements{v.Requirements()} + }), + }, nil +} + +func (a AL2023) resolvePath(architecture, variant, k8sVersion, amiVersion string) string { + name := lo.Ternary( + amiVersion == AMIVersionLatest, + "recommended", + fmt.Sprintf("amazon-eks-node-al2023-%s-%s-%s-%s", architecture, variant, k8sVersion, amiVersion), + ) + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/%s/%s/%s/image_id", k8sVersion, architecture, variant, name) } -func (a AL2023) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ []*cloudprovider.InstanceType, customUserData *string, instanceStorePolicy *v1beta1.InstanceStorePolicy) bootstrap.Bootstrapper { +func (a AL2023) UserData(kubeletConfig *v1.KubeletConfiguration, taints []corev1.Taint, labels map[string]string, caBundle *string, _ []*cloudprovider.InstanceType, customUserData *string, instanceStorePolicy *v1.InstanceStorePolicy) bootstrap.Bootstrapper { return bootstrap.Nodeadm{ Options: bootstrap.Options{ ClusterName: a.Options.ClusterName, @@ -67,8 +93,8 @@ func (a AL2023) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints } // DefaultBlockDeviceMappings returns the default block device mappings for the AMI Family -func (a AL2023) DefaultBlockDeviceMappings() []*v1beta1.BlockDeviceMapping { - return []*v1beta1.BlockDeviceMapping{{ +func (a AL2023) DefaultBlockDeviceMappings() []*v1.BlockDeviceMapping { + return []*v1.BlockDeviceMapping{{ DeviceName: a.EphemeralBlockDevice(), EBS: &DefaultEBS, }} diff --git a/pkg/providers/amifamily/ami.go b/pkg/providers/amifamily/ami.go index fd89ced04dcb..7fae92e8b086 100644 --- a/pkg/providers/amifamily/ami.go +++ b/pkg/providers/amifamily/ami.go @@ -17,245 +17,106 @@ package amifamily import ( "context" "fmt" - "sort" - "strings" + "sync" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" - "github.com/aws/aws-sdk-go/service/ssm" - "github.com/aws/aws-sdk-go/service/ssm/ssmiface" "github.com/mitchellh/hashstructure/v2" "github.com/patrickmn/go-cache" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/log" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/version" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/scheduling" "sigs.k8s.io/karpenter/pkg/utils/pretty" + + "github.com/aws/karpenter-provider-aws/pkg/providers/ssm" ) type Provider interface { - Get(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, options *Options) (AMIs, error) + List(ctx context.Context, nodeClass *v1.EC2NodeClass) (AMIs, error) } type DefaultProvider struct { + sync.Mutex cache *cache.Cache - ssm ssmiface.SSMAPI ec2api ec2iface.EC2API cm *pretty.ChangeMonitor versionProvider version.Provider + ssmProvider ssm.Provider } -type AMI struct { - Name string - AmiID string - CreationDate string - Requirements scheduling.Requirements -} - -type AMIs []AMI - -// Sort orders the AMIs by creation date in descending order. -// If creation date is nil or two AMIs have the same creation date, the AMIs will be sorted by name in ascending order. -func (a AMIs) Sort() { - sort.Slice(a, func(i, j int) bool { - itime, _ := time.Parse(time.RFC3339, a[i].CreationDate) - jtime, _ := time.Parse(time.RFC3339, a[j].CreationDate) - if itime.Unix() != jtime.Unix() { - return itime.Unix() > jtime.Unix() - } - if a[i].Name != a[j].Name { - return a[i].Name < a[j].Name - } - iHash, _ := hashstructure.Hash(a[i].Requirements, hashstructure.FormatV2, &hashstructure.HashOptions{}) - jHash, _ := hashstructure.Hash(a[i].Requirements, hashstructure.FormatV2, &hashstructure.HashOptions{}) - return iHash < jHash - }) -} - -func (a AMIs) String() string { - var sb strings.Builder - ids := lo.Map(a, func(a AMI, _ int) string { return a.AmiID }) - if len(a) > 25 { - sb.WriteString(strings.Join(ids[:25], ", ")) - sb.WriteString(fmt.Sprintf(" and %d other(s)", len(a)-25)) - } else { - sb.WriteString(strings.Join(ids, ", ")) - } - return sb.String() -} - -// MapToInstanceTypes returns a map of AMIIDs that are the most recent on creationDate to compatible instancetypes -func (a AMIs) MapToInstanceTypes(instanceTypes []*cloudprovider.InstanceType) map[string][]*cloudprovider.InstanceType { - amiIDs := map[string][]*cloudprovider.InstanceType{} - for _, instanceType := range instanceTypes { - for _, ami := range a { - if err := instanceType.Requirements.Compatible(ami.Requirements, scheduling.AllowUndefinedWellKnownLabels); err == nil { - amiIDs[ami.AmiID] = append(amiIDs[ami.AmiID], instanceType) - break - } - } - } - return amiIDs -} - -func NewDefaultProvider(versionProvider version.Provider, ssm ssmiface.SSMAPI, ec2api ec2iface.EC2API, cache *cache.Cache) *DefaultProvider { +func NewDefaultProvider(versionProvider version.Provider, ssmProvider ssm.Provider, ec2api ec2iface.EC2API, cache *cache.Cache) *DefaultProvider { return &DefaultProvider{ cache: cache, - ssm: ssm, ec2api: ec2api, cm: pretty.NewChangeMonitor(), versionProvider: versionProvider, + ssmProvider: ssmProvider, } } // Get Returning a list of AMIs with its associated requirements -func (p *DefaultProvider) Get(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, options *Options) (AMIs, error) { - var err error - var amis AMIs - if len(nodeClass.Spec.AMISelectorTerms) == 0 { - amis, err = p.getDefaultAMIs(ctx, nodeClass, options) - if err != nil { - return nil, err - } - } else { - amis, err = p.getAMIs(ctx, nodeClass.Spec.AMISelectorTerms) - if err != nil { - return nil, err - } +func (p *DefaultProvider) List(ctx context.Context, nodeClass *v1.EC2NodeClass) (AMIs, error) { + p.Lock() + defer p.Unlock() + queries, err := p.DescribeImageQueries(ctx, nodeClass) + if err != nil { + return nil, fmt.Errorf("getting AMI queries, %w", err) + } + amis, err := p.amis(ctx, queries) + if err != nil { + return nil, err } amis.Sort() - if p.cm.HasChanged(fmt.Sprintf("amis/%s", nodeClass.Name), amis) { - logging.FromContext(ctx).With("ids", amis, "count", len(amis)).Debugf("discovered amis") + uniqueAMIs := lo.Uniq(lo.Map(amis, func(a AMI, _ int) string { return a.AmiID })) + if p.cm.HasChanged(fmt.Sprintf("amis/%s", nodeClass.Name), uniqueAMIs) { + log.FromContext(ctx).WithValues( + "ids", uniqueAMIs).V(1).Info("discovered amis") } return amis, nil } -func (p *DefaultProvider) getDefaultAMIs(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, options *Options) (res AMIs, err error) { - if images, ok := p.cache.Get(lo.FromPtr(nodeClass.Spec.AMIFamily)); ok { - return images.(AMIs), nil - } - amiFamily := GetAMIFamily(nodeClass.Spec.AMIFamily, options) - kubernetesVersion, err := p.versionProvider.Get(ctx) - if err != nil { - return nil, fmt.Errorf("getting kubernetes version %w", err) - } - defaultAMIs := amiFamily.DefaultAMIs(kubernetesVersion) - for _, ami := range defaultAMIs { - if id, err := p.resolveSSMParameter(ctx, ami.Query); err != nil { - logging.FromContext(ctx).With("query", ami.Query).Errorf("discovering amis from ssm, %s", err) - } else { - res = append(res, AMI{AmiID: id, Requirements: ami.Requirements}) - } - } - // Resolve Name and CreationDate information into the DefaultAMIs - if err = p.ec2api.DescribeImagesPagesWithContext(ctx, &ec2.DescribeImagesInput{ - Filters: []*ec2.Filter{{Name: aws.String("image-id"), Values: aws.StringSlice(lo.Map(res, func(a AMI, _ int) string { return a.AmiID }))}}, - MaxResults: aws.Int64(500), - }, func(page *ec2.DescribeImagesOutput, _ bool) bool { - for i := range page.Images { - for j := range res { - if res[j].AmiID == aws.StringValue(page.Images[i].ImageId) { - res[j].Name = aws.StringValue(page.Images[i].Name) - res[j].CreationDate = aws.StringValue(page.Images[i].CreationDate) - } - } +func (p *DefaultProvider) DescribeImageQueries(ctx context.Context, nodeClass *v1.EC2NodeClass) ([]DescribeImageQuery, error) { + // Aliases are mutually exclusive, both on the term level and field level within a term. + // This is enforced by a CEL validation, we will treat this as an invariant. + if term, ok := lo.Find(nodeClass.Spec.AMISelectorTerms, func(term v1.AMISelectorTerm) bool { + return term.Alias != "" + }); ok { + kubernetesVersion, err := p.versionProvider.Get(ctx) + if err != nil { + return nil, fmt.Errorf("getting kubernetes version, %w", err) } - return true - }); err != nil { - return nil, fmt.Errorf("describing images, %w", err) - } - p.cache.SetDefault(lo.FromPtr(nodeClass.Spec.AMIFamily), res) - return res, nil -} - -func (p *DefaultProvider) resolveSSMParameter(ctx context.Context, ssmQuery string) (string, error) { - output, err := p.ssm.GetParameterWithContext(ctx, &ssm.GetParameterInput{Name: aws.String(ssmQuery)}) - if err != nil { - return "", fmt.Errorf("getting ssm parameter %q, %w", ssmQuery, err) - } - ami := aws.StringValue(output.Parameter.Value) - return ami, nil -} - -func (p *DefaultProvider) getAMIs(ctx context.Context, terms []v1beta1.AMISelectorTerm) (AMIs, error) { - filterAndOwnerSets := GetFilterAndOwnerSets(terms) - hash, err := hashstructure.Hash(filterAndOwnerSets, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}) - if err != nil { - return nil, err - } - if images, ok := p.cache.Get(fmt.Sprintf("%d", hash)); ok { - return images.(AMIs), nil - } - images := map[uint64]AMI{} - for _, filtersAndOwners := range filterAndOwnerSets { - if err = p.ec2api.DescribeImagesPagesWithContext(ctx, &ec2.DescribeImagesInput{ - // Don't include filters in the Describe Images call as EC2 API doesn't allow empty filters. - Filters: lo.Ternary(len(filtersAndOwners.Filters) > 0, filtersAndOwners.Filters, nil), - Owners: lo.Ternary(len(filtersAndOwners.Owners) > 0, aws.StringSlice(filtersAndOwners.Owners), nil), - MaxResults: aws.Int64(500), - }, func(page *ec2.DescribeImagesOutput, _ bool) bool { - for i := range page.Images { - reqs := p.getRequirementsFromImage(page.Images[i]) - if !v1beta1.WellKnownArchitectures.Has(reqs.Get(v1.LabelArchStable).Any()) { - continue - } - reqsHash := lo.Must(hashstructure.Hash(reqs.NodeSelectorRequirements(), hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true})) - // If the proposed image is newer, store it so that we can return it - if v, ok := images[reqsHash]; ok { - candidateCreationTime, _ := time.Parse(time.RFC3339, lo.FromPtr(page.Images[i].CreationDate)) - existingCreationTime, _ := time.Parse(time.RFC3339, v.CreationDate) - if existingCreationTime == candidateCreationTime && lo.FromPtr(page.Images[i].Name) < v.Name { - continue - } - if candidateCreationTime.Unix() < existingCreationTime.Unix() { - continue - } - } - images[reqsHash] = AMI{ - Name: lo.FromPtr(page.Images[i].Name), - AmiID: lo.FromPtr(page.Images[i].ImageId), - CreationDate: lo.FromPtr(page.Images[i].CreationDate), - Requirements: reqs, - } - } - return true - }); err != nil { - return nil, fmt.Errorf("describing images, %w", err) + amiFamily := GetAMIFamily(v1.AMIFamilyFromAlias(term.Alias), nil) + query, err := amiFamily.DescribeImageQuery(ctx, p.ssmProvider, kubernetesVersion, v1.AMIVersionFromAlias(term.Alias)) + if err != nil { + return []DescribeImageQuery{}, err } + return []DescribeImageQuery{query}, nil } - p.cache.SetDefault(fmt.Sprintf("%d", hash), AMIs(lo.Values(images))) - return lo.Values(images), nil -} - -type FiltersAndOwners struct { - Filters []*ec2.Filter - Owners []string -} -func GetFilterAndOwnerSets(terms []v1beta1.AMISelectorTerm) (res []FiltersAndOwners) { idFilter := &ec2.Filter{Name: aws.String("image-id")} - for _, term := range terms { + queries := []DescribeImageQuery{} + for _, term := range nodeClass.Spec.AMISelectorTerms { switch { case term.ID != "": idFilter.Values = append(idFilter.Values, aws.String(term.ID)) default: - elem := FiltersAndOwners{ + query := DescribeImageQuery{ Owners: lo.Ternary(term.Owner != "", []string{term.Owner}, []string{}), } if term.Name != "" { // Default owners to self,amazon to ensure Karpenter only discovers cross-account AMIs if the user specifically allows it. // Removing this default would cause Karpenter to discover publicly shared AMIs passing the name filter. - elem = FiltersAndOwners{ + query = DescribeImageQuery{ Owners: lo.Ternary(term.Owner != "", []string{term.Owner}, []string{"self", "amazon"}), } - elem.Filters = append(elem.Filters, &ec2.Filter{ + query.Filters = append(query.Filters, &ec2.Filter{ Name: aws.String("name"), Values: aws.StringSlice([]string{term.Name}), }) @@ -263,33 +124,90 @@ func GetFilterAndOwnerSets(terms []v1beta1.AMISelectorTerm) (res []FiltersAndOwn } for k, v := range term.Tags { if v == "*" { - elem.Filters = append(elem.Filters, &ec2.Filter{ + query.Filters = append(query.Filters, &ec2.Filter{ Name: aws.String("tag-key"), Values: []*string{aws.String(k)}, }) } else { - elem.Filters = append(elem.Filters, &ec2.Filter{ + query.Filters = append(query.Filters, &ec2.Filter{ Name: aws.String(fmt.Sprintf("tag:%s", k)), Values: []*string{aws.String(v)}, }) } } - res = append(res, elem) + queries = append(queries, query) } } if len(idFilter.Values) > 0 { - res = append(res, FiltersAndOwners{Filters: []*ec2.Filter{idFilter}}) + queries = append(queries, DescribeImageQuery{Filters: []*ec2.Filter{idFilter}}) } - return res + return queries, nil } -func (p *DefaultProvider) getRequirementsFromImage(ec2Image *ec2.Image) scheduling.Requirements { - requirements := scheduling.NewRequirements() - // Always add the architecture of an image as a requirement, irrespective of what's specified in EC2 tags. - architecture := *ec2Image.Architecture - if value, ok := v1beta1.AWSToKubeArchitectures[architecture]; ok { - architecture = value +//nolint:gocyclo +func (p *DefaultProvider) amis(ctx context.Context, queries []DescribeImageQuery) (AMIs, error) { + hash, err := hashstructure.Hash(queries, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}) + if err != nil { + return nil, err } - requirements.Add(scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, architecture)) - return requirements + if images, ok := p.cache.Get(fmt.Sprintf("%d", hash)); ok { + // Ensure what's returned from this function is a deep-copy of AMIs so alterations + // to the data don't affect the original + return append(AMIs{}, images.(AMIs)...), nil + } + images := map[uint64]AMI{} + for _, query := range queries { + if err = p.ec2api.DescribeImagesPagesWithContext(ctx, query.DescribeImagesInput(), func(page *ec2.DescribeImagesOutput, _ bool) bool { + for _, image := range page.Images { + arch, ok := v1.AWSToKubeArchitectures[lo.FromPtr(image.Architecture)] + if !ok { + continue + } + // Each image may have multiple associated sets of requirements. For example, an image may be compatible with Neuron instances + // and GPU instances. In that case, we'll have a set of requirements for each, and will create one "image" for each. + for _, reqs := range query.RequirementsForImageWithArchitecture(lo.FromPtr(image.ImageId), arch) { + // If we already have an image with the same set of requirements, but this image is newer, replace the previous image. + reqsHash := lo.Must(hashstructure.Hash(reqs.NodeSelectorRequirements(), hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true})) + if v, ok := images[reqsHash]; ok { + candidateCreationTime, _ := time.Parse(time.RFC3339, lo.FromPtr(image.CreationDate)) + existingCreationTime, _ := time.Parse(time.RFC3339, v.CreationDate) + if existingCreationTime == candidateCreationTime && lo.FromPtr(image.Name) < v.Name { + continue + } + if candidateCreationTime.Unix() < existingCreationTime.Unix() { + continue + } + } + images[reqsHash] = AMI{ + Name: lo.FromPtr(image.Name), + AmiID: lo.FromPtr(image.ImageId), + CreationDate: lo.FromPtr(image.CreationDate), + Requirements: reqs, + } + } + } + return true + }); err != nil { + return nil, fmt.Errorf("describing images, %w", err) + } + } + p.cache.SetDefault(fmt.Sprintf("%d", hash), AMIs(lo.Values(images))) + return lo.Values(images), nil +} + +// MapToInstanceTypes returns a map of AMIIDs that are the most recent on creationDate to compatible instancetypes +func MapToInstanceTypes(instanceTypes []*cloudprovider.InstanceType, amis []v1.AMI) map[string][]*cloudprovider.InstanceType { + amiIDs := map[string][]*cloudprovider.InstanceType{} + for _, instanceType := range instanceTypes { + for _, ami := range amis { + if err := instanceType.Requirements.Compatible( + scheduling.NewNodeSelectorRequirements(ami.Requirements...), + scheduling.AllowUndefinedWellKnownLabels, + ); err == nil { + amiIDs[ami.ID] = append(amiIDs[ami.ID], instanceType) + break + } + } + } + return amiIDs } diff --git a/pkg/providers/amifamily/bootstrap/bootstrap.go b/pkg/providers/amifamily/bootstrap/bootstrap.go index ac39772d2c53..052ac10a2510 100644 --- a/pkg/providers/amifamily/bootstrap/bootstrap.go +++ b/pkg/providers/amifamily/bootstrap/bootstrap.go @@ -20,13 +20,10 @@ import ( "strings" "github.com/samber/lo" - core "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "knative.dev/pkg/ptr" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" ) // Options is the node bootstrapping parameters passed from Karpenter to the provisioning node @@ -34,14 +31,14 @@ type Options struct { ClusterName string ClusterEndpoint string ClusterCIDR *string - KubeletConfig *corev1beta1.KubeletConfiguration - Taints []core.Taint `hash:"set"` + KubeletConfig *v1.KubeletConfiguration + Taints []corev1.Taint `hash:"set"` Labels map[string]string `hash:"set"` CABundle *string AWSENILimitedPodDensity bool ContainerRuntime *string CustomUserData *string - InstanceStorePolicy *v1beta1.InstanceStorePolicy + InstanceStorePolicy *v1.InstanceStorePolicy } func (o Options) kubeletExtraArgs() (args []string) { @@ -51,10 +48,10 @@ func (o Options) kubeletExtraArgs() (args []string) { return lo.Compact(args) } if o.KubeletConfig.MaxPods != nil { - args = append(args, fmt.Sprintf("--max-pods=%d", ptr.Int32Value(o.KubeletConfig.MaxPods))) + args = append(args, fmt.Sprintf("--max-pods=%d", lo.FromPtr(o.KubeletConfig.MaxPods))) } if o.KubeletConfig.PodsPerCore != nil { - args = append(args, fmt.Sprintf("--pods-per-core=%d", ptr.Int32Value(o.KubeletConfig.PodsPerCore))) + args = append(args, fmt.Sprintf("--pods-per-core=%d", lo.FromPtr(o.KubeletConfig.PodsPerCore))) } // We have to convert some of these maps so that their values return the correct string args = append(args, joinParameterArgs("--system-reserved", o.KubeletConfig.SystemReserved, "=")) @@ -64,13 +61,13 @@ func (o Options) kubeletExtraArgs() (args []string) { args = append(args, joinParameterArgs("--eviction-soft-grace-period", lo.MapValues(o.KubeletConfig.EvictionSoftGracePeriod, func(v metav1.Duration, _ string) string { return v.Duration.String() }), "=")) if o.KubeletConfig.EvictionMaxPodGracePeriod != nil { - args = append(args, fmt.Sprintf("--eviction-max-pod-grace-period=%d", ptr.Int32Value(o.KubeletConfig.EvictionMaxPodGracePeriod))) + args = append(args, fmt.Sprintf("--eviction-max-pod-grace-period=%d", lo.FromPtr(o.KubeletConfig.EvictionMaxPodGracePeriod))) } if o.KubeletConfig.ImageGCHighThresholdPercent != nil { - args = append(args, fmt.Sprintf("--image-gc-high-threshold=%d", ptr.Int32Value(o.KubeletConfig.ImageGCHighThresholdPercent))) + args = append(args, fmt.Sprintf("--image-gc-high-threshold=%d", lo.FromPtr(o.KubeletConfig.ImageGCHighThresholdPercent))) } if o.KubeletConfig.ImageGCLowThresholdPercent != nil { - args = append(args, fmt.Sprintf("--image-gc-low-threshold=%d", ptr.Int32Value(o.KubeletConfig.ImageGCLowThresholdPercent))) + args = append(args, fmt.Sprintf("--image-gc-low-threshold=%d", lo.FromPtr(o.KubeletConfig.ImageGCLowThresholdPercent))) } if o.KubeletConfig.CPUCFSQuota != nil { args = append(args, fmt.Sprintf("--cpu-cfs-quota=%t", lo.FromPtr(o.KubeletConfig.CPUCFSQuota))) @@ -79,12 +76,9 @@ func (o Options) kubeletExtraArgs() (args []string) { } func (o Options) nodeTaintArg() string { - if len(o.Taints) == 0 { - return "" - } var taintStrings []string for _, taint := range o.Taints { - taintStrings = append(taintStrings, fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect)) + taintStrings = append(taintStrings, taint.ToString()) } return fmt.Sprintf("--register-with-taints=%q", strings.Join(taintStrings, ",")) } diff --git a/pkg/providers/amifamily/bootstrap/bottlerocket.go b/pkg/providers/amifamily/bootstrap/bottlerocket.go index c1b2d8b66962..f7dbddd1b9e3 100644 --- a/pkg/providers/amifamily/bootstrap/bottlerocket.go +++ b/pkg/providers/amifamily/bootstrap/bottlerocket.go @@ -19,8 +19,6 @@ import ( "fmt" "strconv" - "knative.dev/pkg/ptr" - "github.com/imdario/mergo" "github.com/samber/lo" @@ -48,7 +46,7 @@ func (b Bottlerocket) Script() (string, error) { // Backwards compatibility for AWSENILimitedPodDensity flag if b.KubeletConfig != nil && b.KubeletConfig.MaxPods != nil { - s.Settings.Kubernetes.MaxPods = aws.Int(int(ptr.Int32Value(b.KubeletConfig.MaxPods))) + s.Settings.Kubernetes.MaxPods = aws.Int(int(lo.FromPtr(b.KubeletConfig.MaxPods))) } else if !b.AWSENILimitedPodDensity { s.Settings.Kubernetes.MaxPods = aws.Int(110) } diff --git a/pkg/providers/amifamily/bootstrap/bottlerocketsettings.go b/pkg/providers/amifamily/bootstrap/bottlerocketsettings.go index f2f59f0de5e6..fe6c44d4d7f1 100644 --- a/pkg/providers/amifamily/bootstrap/bottlerocketsettings.go +++ b/pkg/providers/amifamily/bootstrap/bottlerocketsettings.go @@ -69,11 +69,15 @@ type BottlerocketKubernetes struct { CPUManagerPolicy *string `toml:"cpu-manager-policy,omitempty"` CPUManagerReconcilePeriod *string `toml:"cpu-manager-reconcile-period,omitempty"` TopologyManagerScope *string `toml:"topology-manager-scope,omitempty"` + TopologyManagerPolicy *string `toml:"topology-manager-policy,omitempty"` ImageGCHighThresholdPercent *string `toml:"image-gc-high-threshold-percent,omitempty"` ImageGCLowThresholdPercent *string `toml:"image-gc-low-threshold-percent,omitempty"` CPUCFSQuota *bool `toml:"cpu-cfs-quota-enforced,omitempty"` ShutdownGracePeriod *string `toml:"shutdown-grace-period,omitempty"` ShutdownGracePeriodForCriticalPods *string `toml:"shutdown-grace-period-for-critical-pods,omitempty"` + ClusterDomain *string `toml:"cluster-domain,omitempty"` + SeccompDefault *bool `toml:"seccomp-default,omitempty"` + PodPidsLimit *int `toml:"pod-pids-limit,omitempty"` } type BottlerocketStaticPod struct { diff --git a/pkg/providers/amifamily/bootstrap/eksbootstrap.go b/pkg/providers/amifamily/bootstrap/eksbootstrap.go index baf874a1133d..9c0a53ccf2fe 100644 --- a/pkg/providers/amifamily/bootstrap/eksbootstrap.go +++ b/pkg/providers/amifamily/bootstrap/eksbootstrap.go @@ -29,7 +29,7 @@ import ( "github.com/samber/lo" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" ) type EKS struct { @@ -77,7 +77,7 @@ func (e EKS) eksBootstrapScript() string { if args := e.kubeletExtraArgs(); len(args) > 0 { userData.WriteString(fmt.Sprintf(" \\\n--kubelet-extra-args '%s'", strings.Join(args, " "))) } - if lo.FromPtr(e.InstanceStorePolicy) == v1beta1.InstanceStorePolicyRAID0 { + if lo.FromPtr(e.InstanceStorePolicy) == v1.InstanceStorePolicyRAID0 { userData.WriteString(" \\\n--local-disks raid0") } return userData.String() diff --git a/pkg/providers/amifamily/bootstrap/mime/suite_test.go b/pkg/providers/amifamily/bootstrap/mime/suite_test.go index e0cf9fc820e6..58b2cc9325b9 100644 --- a/pkg/providers/amifamily/bootstrap/mime/suite_test.go +++ b/pkg/providers/amifamily/bootstrap/mime/suite_test.go @@ -23,8 +23,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "sigs.k8s.io/karpenter/pkg/utils/testing" + "github.com/samber/lo" - . "knative.dev/pkg/logging/testing" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap/mime" ) diff --git a/pkg/providers/amifamily/bootstrap/nodeadm.go b/pkg/providers/amifamily/bootstrap/nodeadm.go index 8c82268199c6..afd128e11b0e 100644 --- a/pkg/providers/amifamily/bootstrap/nodeadm.go +++ b/pkg/providers/amifamily/bootstrap/nodeadm.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/yaml" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap/mime" ) @@ -45,10 +45,10 @@ func (n Nodeadm) Script() (string, error) { if err != nil { return "", fmt.Errorf("parsing custom UserData, %w", err) } - mimeArchive := mime.Archive(append([]mime.Entry{{ + mimeArchive := mime.Archive(append(customEntries, mime.Entry{ ContentType: mime.ContentTypeNodeConfig, Content: nodeConfigYAML, - }}, customEntries...)) + })) userData, err := mimeArchive.Serialize() if err != nil { return "", err @@ -83,7 +83,7 @@ func (n Nodeadm) getNodeConfigYAML() (string, error) { } else { return "", cloudprovider.NewNodeClassNotReadyError(fmt.Errorf("resolving cluster CIDR")) } - if lo.FromPtr(n.InstanceStorePolicy) == v1beta1.InstanceStorePolicyRAID0 { + if lo.FromPtr(n.InstanceStorePolicy) == v1.InstanceStorePolicyRAID0 { config.Spec.Instance.LocalStorage.Strategy = admv1alpha1.LocalStorageRAID0 } inlineConfig, err := n.generateInlineKubeletConfiguration() @@ -115,10 +115,8 @@ func (n Nodeadm) generateInlineKubeletConfiguration() (map[string]runtime.RawExt if err != nil { return nil, err } - if len(n.Taints) != 0 { - kubeConfigMap["registerWithTaints"] = runtime.RawExtension{ - Raw: lo.Must(json.Marshal(n.Taints)), - } + kubeConfigMap["registerWithTaints"] = runtime.RawExtension{ + Raw: lo.Must(json.Marshal(n.Taints)), } return kubeConfigMap, nil } diff --git a/pkg/providers/amifamily/bottlerocket.go b/pkg/providers/amifamily/bottlerocket.go index 5daef92e6a10..a6bfe1c3bd15 100644 --- a/pkg/providers/amifamily/bottlerocket.go +++ b/pkg/providers/amifamily/bottlerocket.go @@ -15,20 +15,22 @@ limitations under the License. package amifamily import ( + "context" "fmt" + "strings" "github.com/samber/lo" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap" + "github.com/aws/karpenter-provider-aws/pkg/providers/ssm" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/scheduling" "github.com/aws/aws-sdk-go/aws" - v1 "k8s.io/api/core/v1" + "github.com/aws/aws-sdk-go/service/ec2" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) @@ -37,58 +39,40 @@ type Bottlerocket struct { *Options } -// DefaultAMIs returns the AMI name, and Requirements, with an SSM query -func (b Bottlerocket) DefaultAMIs(version string) []DefaultAMIOutput { - return []DefaultAMIOutput{ - { - Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id", version), - Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), - scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist), - ), - }, - { - Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/x86_64/latest/image_id", version), - Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), - scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpExists), - ), - }, - { - Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/x86_64/latest/image_id", version), - Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), - scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpExists), - ), - }, - { - Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, corev1beta1.ArchitectureArm64), - Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), - scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist), - ), - }, - { - Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/%s/latest/image_id", version, corev1beta1.ArchitectureArm64), - Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), - scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpExists), - ), - }, - { - Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/%s/latest/image_id", version, corev1beta1.ArchitectureArm64), - Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), - scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpExists), - ), - }, +func (b Bottlerocket) DescribeImageQuery(ctx context.Context, ssmProvider ssm.Provider, k8sVersion string, amiVersion string) (DescribeImageQuery, error) { + // Bottlerocket AMIs versions are prefixed with a v on GitHub, but not in the SSM path. We should accept both. + trimmedAMIVersion := strings.TrimLeft(amiVersion, "v") + ids := map[string][]Variant{} + for path, variants := range map[string][]Variant{ + fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/%s/image_id", k8sVersion, trimmedAMIVersion): {VariantStandard}, + fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/arm64/%s/image_id", k8sVersion, trimmedAMIVersion): {VariantStandard}, + fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/x86_64/%s/image_id", k8sVersion, trimmedAMIVersion): {VariantNeuron, VariantNvidia}, + fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/arm64/%s/image_id", k8sVersion, trimmedAMIVersion): {VariantNeuron, VariantNvidia}, + } { + imageID, err := ssmProvider.Get(ctx, path) + if err != nil { + continue + } + ids[imageID] = variants + } + // Failed to discover any AMIs, we should short circuit AMI discovery + if len(ids) == 0 { + return DescribeImageQuery{}, fmt.Errorf(`failed to discover any AMIs for alias "bottlerocket@%s"`, amiVersion) } + + return DescribeImageQuery{ + Filters: []*ec2.Filter{{ + Name: lo.ToPtr("image-id"), + Values: lo.ToSlicePtr(lo.Keys(ids)), + }}, + KnownRequirements: lo.MapValues(ids, func(variants []Variant, _ string) []scheduling.Requirements { + return lo.Map(variants, func(v Variant, _ int) scheduling.Requirements { return v.Requirements() }) + }), + }, nil } // UserData returns the default userdata script for the AMI Family -func (b Bottlerocket) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ []*cloudprovider.InstanceType, customUserData *string, _ *v1beta1.InstanceStorePolicy) bootstrap.Bootstrapper { +func (b Bottlerocket) UserData(kubeletConfig *v1.KubeletConfiguration, taints []corev1.Taint, labels map[string]string, caBundle *string, _ []*cloudprovider.InstanceType, customUserData *string, _ *v1.InstanceStorePolicy) bootstrap.Bootstrapper { return bootstrap.Bottlerocket{ Options: bootstrap.Options{ ClusterName: b.Options.ClusterName, @@ -103,10 +87,10 @@ func (b Bottlerocket) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, } // DefaultBlockDeviceMappings returns the default block device mappings for the AMI Family -func (b Bottlerocket) DefaultBlockDeviceMappings() []*v1beta1.BlockDeviceMapping { +func (b Bottlerocket) DefaultBlockDeviceMappings() []*v1.BlockDeviceMapping { xvdaEBS := DefaultEBS xvdaEBS.VolumeSize = lo.ToPtr(resource.MustParse("4Gi")) - return []*v1beta1.BlockDeviceMapping{ + return []*v1.BlockDeviceMapping{ { DeviceName: aws.String("/dev/xvda"), EBS: &xvdaEBS, diff --git a/pkg/providers/amifamily/custom.go b/pkg/providers/amifamily/custom.go index f76f71c82a32..6448cad83997 100644 --- a/pkg/providers/amifamily/custom.go +++ b/pkg/providers/amifamily/custom.go @@ -15,14 +15,16 @@ limitations under the License. package amifamily import ( - v1 "k8s.io/api/core/v1" + "context" + + corev1 "k8s.io/api/core/v1" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/cloudprovider" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap" + "github.com/aws/karpenter-provider-aws/pkg/providers/ssm" ) type Custom struct { @@ -31,7 +33,7 @@ type Custom struct { } // UserData returns the default userdata script for the AMI Family -func (c Custom) UserData(_ *corev1beta1.KubeletConfiguration, _ []v1.Taint, _ map[string]string, _ *string, _ []*cloudprovider.InstanceType, customUserData *string, _ *v1beta1.InstanceStorePolicy) bootstrap.Bootstrapper { +func (c Custom) UserData(_ *v1.KubeletConfiguration, _ []corev1.Taint, _ map[string]string, _ *string, _ []*cloudprovider.InstanceType, customUserData *string, _ *v1.InstanceStorePolicy) bootstrap.Bootstrapper { return bootstrap.Custom{ Options: bootstrap.Options{ CustomUserData: customUserData, @@ -39,11 +41,11 @@ func (c Custom) UserData(_ *corev1beta1.KubeletConfiguration, _ []v1.Taint, _ ma } } -func (c Custom) DefaultAMIs(_ string) []DefaultAMIOutput { - return nil +func (c Custom) DescribeImageQuery(_ context.Context, _ ssm.Provider, _ string, _ string) (DescribeImageQuery, error) { + return DescribeImageQuery{}, nil } -func (c Custom) DefaultBlockDeviceMappings() []*v1beta1.BlockDeviceMapping { +func (c Custom) DefaultBlockDeviceMappings() []*v1.BlockDeviceMapping { // By returning nil, we ensure that EC2 will automatically choose the volumes defined by the AMI // and we don't need to describe the AMI ourselves. return nil diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go index 65adec3a9abb..6ab62d975a34 100644 --- a/pkg/providers/amifamily/resolver.go +++ b/pkg/providers/amifamily/resolver.go @@ -21,21 +21,22 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/imdario/mergo" "github.com/samber/lo" - core "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap" + "github.com/aws/karpenter-provider-aws/pkg/providers/ssm" + "github.com/aws/karpenter-provider-aws/pkg/utils" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/scheduling" ) -var DefaultEBS = v1beta1.BlockDevice{ +var DefaultEBS = v1.BlockDevice{ Encrypted: aws.Bool(true), VolumeType: aws.String(ec2.VolumeTypeGp3), VolumeSize: lo.ToPtr(resource.MustParse("20Gi")), @@ -53,9 +54,9 @@ type Options struct { ClusterCIDR *string InstanceProfile string CABundle *string `hash:"ignore"` - InstanceStorePolicy *v1beta1.InstanceStorePolicy + InstanceStorePolicy *v1.InstanceStorePolicy // Level-triggered fields that may change out of sync. - SecurityGroups []v1beta1.SecurityGroup + SecurityGroups []v1.SecurityGroup Tags map[string]string Labels map[string]string `hash:"ignore"` KubeDNSIP net.IP @@ -67,8 +68,8 @@ type Options struct { type LaunchTemplate struct { *Options UserData bootstrap.Bootstrapper - BlockDeviceMappings []*v1beta1.BlockDeviceMapping - MetadataOptions *v1beta1.MetadataOptions + BlockDeviceMappings []*v1.BlockDeviceMapping + MetadataOptions *v1.MetadataOptions AMIID string InstanceTypes []*cloudprovider.InstanceType `hash:"ignore"` DetailedMonitoring bool @@ -78,10 +79,10 @@ type LaunchTemplate struct { // AMIFamily can be implemented to override the default logic for generating dynamic launch template parameters type AMIFamily interface { - DefaultAMIs(version string) []DefaultAMIOutput - UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []core.Taint, labels map[string]string, caBundle *string, instanceTypes []*cloudprovider.InstanceType, customUserData *string, instanceStorePolicy *v1beta1.InstanceStorePolicy) bootstrap.Bootstrapper - DefaultBlockDeviceMappings() []*v1beta1.BlockDeviceMapping - DefaultMetadataOptions() *v1beta1.MetadataOptions + DescribeImageQuery(ctx context.Context, ssmProvider ssm.Provider, k8sVersion string, amiVersion string) (DescribeImageQuery, error) + UserData(kubeletConfig *v1.KubeletConfiguration, taints []corev1.Taint, labels map[string]string, caBundle *string, instanceTypes []*cloudprovider.InstanceType, customUserData *string, instanceStorePolicy *v1.InstanceStorePolicy) bootstrap.Bootstrapper + DefaultBlockDeviceMappings() []*v1.BlockDeviceMapping + DefaultMetadataOptions() *v1.MetadataOptions EphemeralBlockDevice() *string FeatureFlags() FeatureFlags } @@ -120,18 +121,14 @@ func NewResolver(amiProvider Provider) *Resolver { // Resolve generates launch templates using the static options and dynamically generates launch template parameters. // Multiple ResolvedTemplates are returned based on the instanceTypes passed in to support special AMIs for certain instance types like GPUs. -func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, options *Options) ([]*LaunchTemplate, error) { - amiFamily := GetAMIFamily(nodeClass.Spec.AMIFamily, options) - amis, err := r.amiProvider.Get(ctx, nodeClass, options) - if err != nil { - return nil, err - } - if len(amis) == 0 { +func (r Resolver) Resolve(nodeClass *v1.EC2NodeClass, nodeClaim *karpv1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, options *Options) ([]*LaunchTemplate, error) { + amiFamily := GetAMIFamily(nodeClass.AMIFamily(), options) + if len(nodeClass.Status.AMIs) == 0 { return nil, fmt.Errorf("no amis exist given constraints") } - mappedAMIs := amis.MapToInstanceTypes(instanceTypes) + mappedAMIs := MapToInstanceTypes(instanceTypes, nodeClass.Status.AMIs) if len(mappedAMIs) == 0 { - return nil, fmt.Errorf("no instance types satisfy requirements of amis %v", amis) + return nil, fmt.Errorf("no instance types satisfy requirements of amis %v", lo.Uniq(lo.Map(nodeClass.Status.AMIs, func(a v1.AMI, _ int) string { return a.ID }))) } var resolvedTemplates []*LaunchTemplate for amiID, instanceTypes := range mappedAMIs { @@ -147,8 +144,8 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, paramsToInstanceTypes := lo.GroupBy(instanceTypes, func(instanceType *cloudprovider.InstanceType) launchTemplateParams { return launchTemplateParams{ efaCount: lo.Ternary( - lo.Contains(lo.Keys(nodeClaim.Spec.Resources.Requests), v1beta1.ResourceEFA), - int(lo.ToPtr(instanceType.Capacity[v1beta1.ResourceEFA]).Value()), + lo.Contains(lo.Keys(nodeClaim.Spec.Resources.Requests), v1.ResourceEFA), + int(lo.ToPtr(instanceType.Capacity[v1.ResourceEFA]).Value()), 0, ), maxPods: int(instanceType.Capacity.Pods().Value()), @@ -165,27 +162,25 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, return resolvedTemplates, nil } -func GetAMIFamily(amiFamily *string, options *Options) AMIFamily { - switch aws.StringValue(amiFamily) { - case v1beta1.AMIFamilyBottlerocket: +func GetAMIFamily(amiFamily string, options *Options) AMIFamily { + switch amiFamily { + case v1.AMIFamilyBottlerocket: return &Bottlerocket{Options: options} - case v1beta1.AMIFamilyUbuntu: - return &Ubuntu{Options: options} - case v1beta1.AMIFamilyWindows2019: - return &Windows{Options: options, Version: v1beta1.Windows2019, Build: v1beta1.Windows2019Build} - case v1beta1.AMIFamilyWindows2022: - return &Windows{Options: options, Version: v1beta1.Windows2022, Build: v1beta1.Windows2022Build} - case v1beta1.AMIFamilyCustom: + case v1.AMIFamilyWindows2019: + return &Windows{Options: options, Version: v1.Windows2019, Build: v1.Windows2019Build} + case v1.AMIFamilyWindows2022: + return &Windows{Options: options, Version: v1.Windows2022, Build: v1.Windows2022Build} + case v1.AMIFamilyCustom: return &Custom{Options: options} - case v1beta1.AMIFamilyAL2023: + case v1.AMIFamilyAL2023: return &AL2023{Options: options} default: return &AL2{Options: options} } } -func (o Options) DefaultMetadataOptions() *v1beta1.MetadataOptions { - return &v1beta1.MetadataOptions{ +func (o Options) DefaultMetadataOptions() *v1.MetadataOptions { + return &v1.MetadataOptions{ HTTPEndpoint: aws.String(ec2.LaunchTemplateInstanceMetadataEndpointStateEnabled), HTTPProtocolIPv6: aws.String(lo.Ternary(o.KubeDNSIP == nil || o.KubeDNSIP.To4() != nil, ec2.LaunchTemplateInstanceMetadataProtocolIpv6Disabled, ec2.LaunchTemplateInstanceMetadataProtocolIpv6Enabled)), HTTPPutResponseHopLimit: aws.Int64(2), @@ -193,7 +188,7 @@ func (o Options) DefaultMetadataOptions() *v1beta1.MetadataOptions { } } -func (r Resolver) defaultClusterDNS(opts *Options, kubeletConfig *corev1beta1.KubeletConfiguration) *corev1beta1.KubeletConfiguration { +func (r Resolver) defaultClusterDNS(opts *Options, kubeletConfig *v1.KubeletConfiguration) *v1.KubeletConfiguration { if opts.KubeDNSIP == nil { return kubeletConfig } @@ -201,7 +196,7 @@ func (r Resolver) defaultClusterDNS(opts *Options, kubeletConfig *corev1beta1.Ku return kubeletConfig } if kubeletConfig == nil { - return &corev1beta1.KubeletConfiguration{ + return &v1.KubeletConfiguration{ ClusterDNS: []string{opts.KubeDNSIP.String()}, } } @@ -210,22 +205,33 @@ func (r Resolver) defaultClusterDNS(opts *Options, kubeletConfig *corev1beta1.Ku return newKubeletConfig } -func (r Resolver) resolveLaunchTemplate(nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, +func (r Resolver) resolveLaunchTemplate(nodeClass *v1.EC2NodeClass, nodeClaim *karpv1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, amiFamily AMIFamily, amiID string, maxPods int, efaCount int, options *Options) (*LaunchTemplate, error) { - kubeletConfig := &corev1beta1.KubeletConfiguration{} - if nodeClaim.Spec.Kubelet != nil { - if err := mergo.Merge(kubeletConfig, nodeClaim.Spec.Kubelet); err != nil { - return nil, err - } + kubeletConfig, err := utils.GetKubeletConfigurationWithNodeClaim(nodeClaim, nodeClass) + if err != nil { + return nil, fmt.Errorf("resolving kubelet configuration, %w", err) + } + if kubeletConfig == nil { + kubeletConfig = &v1.KubeletConfiguration{} } if kubeletConfig.MaxPods == nil { kubeletConfig.MaxPods = lo.ToPtr(int32(maxPods)) } + taints := lo.Flatten([][]corev1.Taint{ + nodeClaim.Spec.Taints, + nodeClaim.Spec.StartupTaints, + }) + if _, found := lo.Find(taints, func(t corev1.Taint) bool { + return t.MatchTaint(&karpv1.UnregisteredNoExecuteTaint) + }); !found { + taints = append(taints, karpv1.UnregisteredNoExecuteTaint) + } + resolved := &LaunchTemplate{ Options: options, UserData: amiFamily.UserData( r.defaultClusterDNS(options, kubeletConfig), - append(nodeClaim.Spec.Taints, nodeClaim.Spec.StartupTaints...), + taints, options.Labels, options.CABundle, instanceTypes, diff --git a/pkg/providers/amifamily/suite_test.go b/pkg/providers/amifamily/suite_test.go index 7dfc5ebe0d0b..8e9110322e0d 100644 --- a/pkg/providers/amifamily/suite_test.go +++ b/pkg/providers/amifamily/suite_test.go @@ -18,25 +18,29 @@ import ( "context" "fmt" "sort" + "sync" "testing" "time" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "sigs.k8s.io/karpenter/pkg/utils/testing" + "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - . "knative.dev/pkg/logging/testing" + corev1 "k8s.io/api/core/v1" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" "sigs.k8s.io/karpenter/pkg/scheduling" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/test" @@ -45,7 +49,7 @@ import ( var ctx context.Context var env *coretest.Environment var awsEnv *test.Environment -var nodeClass *v1beta1.EC2NodeClass +var nodeClass *v1.EC2NodeClass func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -61,7 +65,7 @@ const ( ) var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) awsEnv = test.NewEnvironment(ctx, env) @@ -74,7 +78,7 @@ var _ = BeforeEach(func() { { Name: aws.String(amd64AMI), ImageId: aws.String("amd64-ami-id"), - CreationDate: aws.String(time.Now().Format(time.RFC3339)), + CreationDate: aws.String(time.Time{}.Format(time.RFC3339)), Architecture: aws.String("x86_64"), Tags: []*ec2.Tag{ {Key: aws.String("Name"), Value: aws.String(amd64AMI)}, @@ -84,7 +88,7 @@ var _ = BeforeEach(func() { { Name: aws.String(arm64AMI), ImageId: aws.String("arm64-ami-id"), - CreationDate: aws.String(time.Now().Add(time.Minute).Format(time.RFC3339)), + CreationDate: aws.String(time.Time{}.Add(time.Minute).Format(time.RFC3339)), Architecture: aws.String("arm64"), Tags: []*ec2.Tag{ {Key: aws.String("Name"), Value: aws.String(arm64AMI)}, @@ -94,7 +98,7 @@ var _ = BeforeEach(func() { { Name: aws.String(amd64NvidiaAMI), ImageId: aws.String("amd64-nvidia-ami-id"), - CreationDate: aws.String(time.Now().Add(2 * time.Minute).Format(time.RFC3339)), + CreationDate: aws.String(time.Time{}.Add(2 * time.Minute).Format(time.RFC3339)), Architecture: aws.String("x86_64"), Tags: []*ec2.Tag{ {Key: aws.String("Name"), Value: aws.String(amd64NvidiaAMI)}, @@ -104,7 +108,7 @@ var _ = BeforeEach(func() { { Name: aws.String(arm64NvidiaAMI), ImageId: aws.String("arm64-nvidia-ami-id"), - CreationDate: aws.String(time.Now().Add(2 * time.Minute).Format(time.RFC3339)), + CreationDate: aws.String(time.Time{}.Add(2 * time.Minute).Format(time.RFC3339)), Architecture: aws.String("arm64"), Tags: []*ec2.Tag{ {Key: aws.String("Name"), Value: aws.String(arm64NvidiaAMI)}, @@ -130,96 +134,123 @@ var _ = Describe("AMIProvider", func() { nodeClass = test.EC2NodeClass() }) It("should succeed to resolve AMIs (AL2)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): amd64AMI, fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version): amd64NvidiaAMI, fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id", version): arm64AMI, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(4)) }) It("should succeed to resolve AMIs (AL2023)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2023@latest"}} awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", version): amd64AMI, fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version): arm64AMI, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(2)) }) It("should succeed to resolve AMIs (Bottlerocket)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id", version): amd64AMI, fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/x86_64/latest/image_id", version): amd64NvidiaAMI, fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id", version): arm64AMI, fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/arm64/latest/image_id", version): arm64NvidiaAMI, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(6)) }) - It("should succeed to resolve AMIs (Ubuntu)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyUbuntu - awsEnv.SSMAPI.Parameters = map[string]string{ - fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/amd64/hvm/ebs-gp2/ami-id", version): amd64AMI, - fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/arm64/hvm/ebs-gp2/ami-id", version): arm64AMI, - } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) - Expect(err).ToNot(HaveOccurred()) - Expect(amis).To(HaveLen(2)) - }) It("should succeed to resolve AMIs (Windows2019)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2019 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "windows2019@latest"}} awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-%s/image_id", version): amd64AMI, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(1)) }) It("should succeed to resolve AMIs (Windows2022)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "windows2022@latest"}} awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-%s/image_id", version): amd64AMI, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(1)) }) - It("should succeed to resolve AMIs (Custom)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) - Expect(err).ToNot(HaveOccurred()) - Expect(amis).To(HaveLen(0)) + It("should not cause data races when calling Get() simultaneously", func() { + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + ID: "amd64-ami-id", + }, + { + ID: "arm64-ami-id", + }, + } + wg := sync.WaitGroup{} + for i := 0; i < 10000; i++ { + wg.Add(1) + go func() { + defer wg.Done() + defer GinkgoRecover() + images, err := awsEnv.AMIProvider.List(ctx, nodeClass) + Expect(err).ToNot(HaveOccurred()) + + Expect(images).To(HaveLen(2)) + // Sort everything in parallel and ensure that we don't get data races + images.Sort() + Expect(images).To(BeEquivalentTo([]amifamily.AMI{ + { + Name: arm64AMI, + AmiID: "arm64-ami-id", + CreationDate: time.Time{}.Add(time.Minute).Format(time.RFC3339), + Requirements: scheduling.NewLabelRequirements(map[string]string{ + corev1.LabelArchStable: karpv1.ArchitectureArm64, + }), + }, + { + Name: amd64AMI, + AmiID: "amd64-ami-id", + CreationDate: time.Time{}.Format(time.RFC3339), + Requirements: scheduling.NewLabelRequirements(map[string]string{ + corev1.LabelArchStable: karpv1.ArchitectureAmd64, + }), + }, + })) + }() + } + wg.Wait() }) Context("SSM Alias Missing", func() { It("should succeed to partially resolve AMIs if all SSM aliases don't exist (Al2)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} // No GPU AMI exists here awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): amd64AMI, fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id", version): arm64AMI, } // Only 2 of the requirements sets for the SSM aliases will resolve - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(2)) }) It("should succeed to partially resolve AMIs if all SSM aliases don't exist (AL2023)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2023@latest"}} awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", version): amd64AMI, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(1)) }) It("should succeed to partially resolve AMIs if all SSM aliases don't exist (Bottlerocket)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} // No GPU AMI exists for AM64 here awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id", version): amd64AMI, @@ -227,21 +258,10 @@ var _ = Describe("AMIProvider", func() { fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id", version): arm64AMI, } // Only 4 of the requirements sets for the SSM aliases will resolve - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(4)) }) - It("should succeed to partially resolve AMIs if all SSM aliases don't exist (Ubuntu)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyUbuntu - // No AMD64 AMI exists here - awsEnv.SSMAPI.Parameters = map[string]string{ - fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/arm64/hvm/ebs-gp2/ami-id", version): arm64AMI, - } - // Only 1 of the requirements sets for the SSM aliases will resolve - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) - Expect(err).ToNot(HaveOccurred()) - Expect(amis).To(HaveLen(1)) - }) }) Context("AMI Tag Requirements", func() { var img *ec2.Image @@ -254,8 +274,8 @@ var _ = Describe("AMIProvider", func() { Tags: []*ec2.Tag{ {Key: aws.String("Name"), Value: aws.String(amd64AMI)}, {Key: aws.String("foo"), Value: aws.String("bar")}, - {Key: aws.String(v1.LabelInstanceTypeStable), Value: aws.String("m5.large")}, - {Key: aws.String(v1.LabelTopologyZone), Value: aws.String("test-zone-1a")}, + {Key: aws.String(corev1.LabelInstanceTypeStable), Value: aws.String("m5.large")}, + {Key: aws.String(corev1.LabelTopologyZone), Value: aws.String("test-zone-1a")}, }, } awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ @@ -265,12 +285,12 @@ var _ = Describe("AMIProvider", func() { }) }) It("should succeed to not resolve tags as requirements for NodeClasses", func() { - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ { Tags: map[string]string{"*": "*"}, }, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(1)) Expect(amis).To(ConsistOf(amifamily.AMI{ @@ -278,7 +298,7 @@ var _ = Describe("AMIProvider", func() { AmiID: aws.StringValue(img.ImageId), CreationDate: aws.StringValue(img.CreationDate), Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(corev1.LabelArchStable, corev1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), ), })) }) @@ -287,15 +307,17 @@ var _ = Describe("AMIProvider", func() { // When you tag public or shared resources, the tags you assign are available only to your AWS account; no other AWS account will have access to those tags // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions It("should have empty owners and use tags when prefixes aren't set", func() { - amiSelectorTerms := []v1beta1.AMISelectorTerm{ - { - Tags: map[string]string{ - "Name": "my-ami", - }, + queries, err := awsEnv.AMIProvider.DescribeImageQueries(ctx, &v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: []v1.AMISelectorTerm{{ + Tags: map[string]string{ + "Name": "my-ami", + }, + }}, }, - } - filterAndOwnersSets := amifamily.GetFilterAndOwnerSets(amiSelectorTerms) - ExpectConsistsOfFiltersAndOwners([]amifamily.FiltersAndOwners{ + }) + Expect(err).To(BeNil()) + ExpectConsistsOfAMIQueries([]amifamily.DescribeImageQuery{ { Filters: []*ec2.Filter{ { @@ -305,16 +327,18 @@ var _ = Describe("AMIProvider", func() { }, Owners: []string{}, }, - }, filterAndOwnersSets) + }, queries) }) It("should have default owners and use name when prefixed", func() { - amiSelectorTerms := []v1beta1.AMISelectorTerm{ - { - Name: "my-ami", + queries, err := awsEnv.AMIProvider.DescribeImageQueries(ctx, &v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: []v1.AMISelectorTerm{{ + Name: "my-ami", + }}, }, - } - filterAndOwnersSets := amifamily.GetFilterAndOwnerSets(amiSelectorTerms) - ExpectConsistsOfFiltersAndOwners([]amifamily.FiltersAndOwners{ + }) + Expect(err).To(BeNil()) + ExpectConsistsOfAMIQueries([]amifamily.DescribeImageQuery{ { Filters: []*ec2.Filter{ { @@ -327,19 +351,23 @@ var _ = Describe("AMIProvider", func() { "self", }, }, - }, filterAndOwnersSets) + }, queries) }) It("should not set owners when legacy ids are passed", func() { - amiSelectorTerms := []v1beta1.AMISelectorTerm{ - { - ID: "ami-abcd1234", - }, - { - ID: "ami-cafeaced", + queries, err := awsEnv.AMIProvider.DescribeImageQueries(ctx, &v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: []v1.AMISelectorTerm{ + { + ID: "ami-abcd1234", + }, + { + ID: "ami-cafeaced", + }, + }, }, - } - filterAndOwnersSets := amifamily.GetFilterAndOwnerSets(amiSelectorTerms) - ExpectConsistsOfFiltersAndOwners([]amifamily.FiltersAndOwners{ + }) + Expect(err).To(BeNil()) + ExpectConsistsOfAMIQueries([]amifamily.DescribeImageQuery{ { Filters: []*ec2.Filter{ { @@ -348,40 +376,48 @@ var _ = Describe("AMIProvider", func() { }, }, }, - }, filterAndOwnersSets) + }, queries) }) It("should allow only specifying owners", func() { - amiSelectorTerms := []v1beta1.AMISelectorTerm{ - { - Owner: "abcdef", - }, - { - Owner: "123456789012", + queries, err := awsEnv.AMIProvider.DescribeImageQueries(ctx, &v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: []v1.AMISelectorTerm{ + { + Owner: "abcdef", + }, + { + Owner: "123456789012", + }, + }, }, - } - filterAndOwnersSets := amifamily.GetFilterAndOwnerSets(amiSelectorTerms) - ExpectConsistsOfFiltersAndOwners([]amifamily.FiltersAndOwners{ + }) + Expect(err).To(BeNil()) + ExpectConsistsOfAMIQueries([]amifamily.DescribeImageQuery{ { Owners: []string{"abcdef"}, }, { Owners: []string{"123456789012"}, }, - }, filterAndOwnersSets) + }, queries) }) It("should allow prefixed name and prefixed owners", func() { - amiSelectorTerms := []v1beta1.AMISelectorTerm{ - { - Name: "my-name", - Owner: "0123456789", - }, - { - Name: "my-name", - Owner: "self", + queries, err := awsEnv.AMIProvider.DescribeImageQueries(ctx, &v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: []v1.AMISelectorTerm{ + { + Name: "my-name", + Owner: "0123456789", + }, + { + Name: "my-name", + Owner: "self", + }, + }, }, - } - filterAndOwnersSets := amifamily.GetFilterAndOwnerSets(amiSelectorTerms) - ExpectConsistsOfFiltersAndOwners([]amifamily.FiltersAndOwners{ + }) + Expect(err).To(BeNil()) + ExpectConsistsOfAMIQueries([]amifamily.DescribeImageQuery{ { Owners: []string{"0123456789"}, Filters: []*ec2.Filter{ @@ -400,7 +436,7 @@ var _ = Describe("AMIProvider", func() { }, }, }, - }, filterAndOwnersSets) + }, queries) }) It("should sort amis by creationDate", func() { amis := amifamily.AMIs{ @@ -459,14 +495,72 @@ var _ = Describe("AMIProvider", func() { }, )) }) + It("should sort amis with the same name and creation date consistently", func() { + amis := amifamily.AMIs{ + { + Name: "test-ami-1", + AmiID: "test-ami-4-id", + CreationDate: "2021-08-31T00:10:42.000Z", + Requirements: scheduling.NewRequirements(), + }, + { + Name: "test-ami-1", + AmiID: "test-ami-3-id", + CreationDate: "2021-08-31T00:10:42.000Z", + Requirements: scheduling.NewRequirements(), + }, + { + Name: "test-ami-1", + AmiID: "test-ami-2-id", + CreationDate: "2021-08-31T00:10:42.000Z", + Requirements: scheduling.NewRequirements(), + }, + { + Name: "test-ami-1", + AmiID: "test-ami-1-id", + CreationDate: "2021-08-31T00:10:42.000Z", + Requirements: scheduling.NewRequirements(), + }, + } + + amis.Sort() + Expect(amis).To(Equal( + amifamily.AMIs{ + { + Name: "test-ami-1", + AmiID: "test-ami-1-id", + CreationDate: "2021-08-31T00:10:42.000Z", + Requirements: scheduling.NewRequirements(), + }, + { + Name: "test-ami-1", + AmiID: "test-ami-2-id", + CreationDate: "2021-08-31T00:10:42.000Z", + Requirements: scheduling.NewRequirements(), + }, + { + Name: "test-ami-1", + AmiID: "test-ami-3-id", + CreationDate: "2021-08-31T00:10:42.000Z", + Requirements: scheduling.NewRequirements(), + }, + { + Name: "test-ami-1", + AmiID: "test-ami-4-id", + CreationDate: "2021-08-31T00:10:42.000Z", + Requirements: scheduling.NewRequirements(), + }, + }, + )) + }) }) }) -func ExpectConsistsOfFiltersAndOwners(expected, actual []amifamily.FiltersAndOwners) { +func ExpectConsistsOfAMIQueries(expected, actual []amifamily.DescribeImageQuery) { GinkgoHelper() Expect(actual).To(HaveLen(len(expected))) - for _, list := range [][]amifamily.FiltersAndOwners{expected, actual} { + for _, list := range [][]amifamily.DescribeImageQuery{expected, actual} { for _, elem := range list { for _, f := range elem.Filters { sort.Slice(f.Values, func(i, j int) bool { @@ -479,5 +573,5 @@ func ExpectConsistsOfFiltersAndOwners(expected, actual []amifamily.FiltersAndOwn }) } } - Expect(actual).To(ConsistOf(lo.Map(expected, func(f amifamily.FiltersAndOwners, _ int) interface{} { return f })...)) + Expect(actual).To(ConsistOf(lo.Map(expected, func(q amifamily.DescribeImageQuery, _ int) interface{} { return q })...)) } diff --git a/pkg/providers/amifamily/types.go b/pkg/providers/amifamily/types.go new file mode 100644 index 000000000000..ac37d20b06b0 --- /dev/null +++ b/pkg/providers/amifamily/types.go @@ -0,0 +1,119 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package amifamily + +import ( + "fmt" + "sort" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/karpenter/pkg/scheduling" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" +) + +const ( + // AMIVersionLatest is the version used in EKS aliases to represent the latest version. This maps to different + // values in the SSM path, depending on the AMI type (e.g. "recommended" for AL2/AL2023)). + AMIVersionLatest = "latest" +) + +type AMI struct { + Name string + AmiID string + CreationDate string + Requirements scheduling.Requirements +} + +type AMIs []AMI + +// Sort orders the AMIs by creation date in descending order. +// If creation date is nil or two AMIs have the same creation date, the AMIs will be sorted by ID, which is guaranteed to be unique, in ascending order. +func (a AMIs) Sort() { + sort.Slice(a, func(i, j int) bool { + itime, _ := time.Parse(time.RFC3339, a[i].CreationDate) + jtime, _ := time.Parse(time.RFC3339, a[j].CreationDate) + if itime.Unix() != jtime.Unix() { + return itime.Unix() > jtime.Unix() + } + return a[i].AmiID < a[j].AmiID + }) +} + +type Variant string + +var ( + VariantStandard Variant = "standard" + VariantNvidia Variant = "nvidia" + VariantNeuron Variant = "neuron" +) + +func NewVariant(v string) (Variant, error) { + var wellKnownVariants = sets.New(VariantStandard, VariantNvidia, VariantNeuron) + variant := Variant(v) + if !wellKnownVariants.Has(variant) { + return variant, fmt.Errorf("%q is not a well-known variant", variant) + } + return variant, nil +} + +func (v Variant) Requirements() scheduling.Requirements { + switch v { + case VariantStandard: + return scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelInstanceAcceleratorCount, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceGPUCount, corev1.NodeSelectorOpDoesNotExist), + ) + case VariantNvidia: + return scheduling.NewRequirements(scheduling.NewRequirement(v1.LabelInstanceAcceleratorCount, corev1.NodeSelectorOpExists)) + case VariantNeuron: + return scheduling.NewRequirements(scheduling.NewRequirement(v1.LabelInstanceGPUCount, corev1.NodeSelectorOpExists)) + } + return nil +} + +type DescribeImageQuery struct { + Filters []*ec2.Filter + Owners []string + // KnownRequirements is a map from image IDs to a set of known requirements. + // When discovering image IDs via SSM we know additional requirements which aren't surfaced by ec2:DescribeImage (e.g. GPU / Neuron compatibility) + // Sometimes, an image may have multiple sets of known requirements. For example, the AL2 GPU AMI is compatible with both Neuron and Nvidia GPU + // instances, which means we need a set of requirements for either instance type. + KnownRequirements map[string][]scheduling.Requirements +} + +func (q DescribeImageQuery) DescribeImagesInput() *ec2.DescribeImagesInput { + return &ec2.DescribeImagesInput{ + // Don't include filters in the Describe Images call as EC2 API doesn't allow empty filters. + Filters: lo.Ternary(len(q.Filters) > 0, q.Filters, nil), + Owners: lo.Ternary(len(q.Owners) > 0, lo.ToSlicePtr(q.Owners), nil), + MaxResults: aws.Int64(1000), + } +} + +func (q DescribeImageQuery) RequirementsForImageWithArchitecture(image string, arch string) []scheduling.Requirements { + if knownRequirements, ok := q.KnownRequirements[image]; ok { + return lo.Map(knownRequirements, func(r scheduling.Requirements, _ int) scheduling.Requirements { + r.Add(scheduling.NewRequirement(corev1.LabelArchStable, corev1.NodeSelectorOpIn, arch)) + return r + }) + } + return []scheduling.Requirements{scheduling.NewRequirements(scheduling.NewRequirement(corev1.LabelArchStable, corev1.NodeSelectorOpIn, arch))} +} diff --git a/pkg/providers/amifamily/ubuntu.go b/pkg/providers/amifamily/ubuntu.go index fc90203b3ff7..64ed1f716d1b 100644 --- a/pkg/providers/amifamily/ubuntu.go +++ b/pkg/providers/amifamily/ubuntu.go @@ -18,11 +18,11 @@ import ( "fmt" "github.com/aws/aws-sdk-go/aws" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap" "sigs.k8s.io/karpenter/pkg/cloudprovider" @@ -35,25 +35,26 @@ type Ubuntu struct { } // DefaultAMIs returns the AMI name, and Requirements, with an SSM query +// TODO: This should be removed at v1.1.0 func (u Ubuntu) DefaultAMIs(version string) []DefaultAMIOutput { return []DefaultAMIOutput{ { - Query: fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/%s/hvm/ebs-gp2/ami-id", version, corev1beta1.ArchitectureAmd64), + Query: fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/%s/hvm/ebs-gp2/ami-id", version, karpv1.ArchitectureAmd64), Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(corev1.LabelArchStable, corev1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), ), }, { - Query: fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/%s/hvm/ebs-gp2/ami-id", version, corev1beta1.ArchitectureArm64), + Query: fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/%s/hvm/ebs-gp2/ami-id", version, karpv1.ArchitectureArm64), Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), + scheduling.NewRequirement(corev1.LabelArchStable, corev1.NodeSelectorOpIn, karpv1.ArchitectureArm64), ), }, } } // UserData returns the default userdata script for the AMI Family -func (u Ubuntu) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ []*cloudprovider.InstanceType, customUserData *string, _ *v1beta1.InstanceStorePolicy) bootstrap.Bootstrapper { +func (u Ubuntu) UserData(kubeletConfig *v1.KubeletConfiguration, taints []corev1.Taint, labels map[string]string, caBundle *string, _ []*cloudprovider.InstanceType, customUserData *string, _ *v1.InstanceStorePolicy) bootstrap.Bootstrapper { return bootstrap.EKS{ Options: bootstrap.Options{ ClusterName: u.Options.ClusterName, @@ -68,8 +69,8 @@ func (u Ubuntu) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints } // DefaultBlockDeviceMappings returns the default block device mappings for the AMI Family -func (u Ubuntu) DefaultBlockDeviceMappings() []*v1beta1.BlockDeviceMapping { - return []*v1beta1.BlockDeviceMapping{{ +func (u Ubuntu) DefaultBlockDeviceMappings() []*v1.BlockDeviceMapping { + return []*v1.BlockDeviceMapping{{ DeviceName: u.EphemeralBlockDevice(), EBS: &DefaultEBS, }} diff --git a/pkg/providers/amifamily/windows.go b/pkg/providers/amifamily/windows.go index 33cf9266b844..f1194dcf9d57 100644 --- a/pkg/providers/amifamily/windows.go +++ b/pkg/providers/amifamily/windows.go @@ -15,21 +15,23 @@ limitations under the License. package amifamily import ( + "context" "fmt" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/scheduling" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/samber/lo" "k8s.io/apimachinery/pkg/api/resource" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap" + "github.com/aws/karpenter-provider-aws/pkg/providers/ssm" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" ) @@ -37,25 +39,34 @@ import ( type Windows struct { DefaultFamily *Options + // Version is the major version of Windows Server (2019 or 2022). + // Only the core version of each version is supported by Karpenter, so this field only indicates the year. Version string - Build string + // Build is a specific build code associated with the Version + Build string } -func (w Windows) DefaultAMIs(version string) []DefaultAMIOutput { - return []DefaultAMIOutput{ - { - Query: fmt.Sprintf("/aws/service/ami-windows-latest/Windows_Server-%s-English-%s-EKS_Optimized-%s/image_id", w.Version, v1beta1.WindowsCore, version), - Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), - scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, string(v1.Windows)), - scheduling.NewRequirement(v1.LabelWindowsBuild, v1.NodeSelectorOpIn, w.Build), - ), - }, +func (w Windows) DescribeImageQuery(ctx context.Context, ssmProvider ssm.Provider, k8sVersion string, amiVersion string) (DescribeImageQuery, error) { + imageID, err := ssmProvider.Get(ctx, fmt.Sprintf("/aws/service/ami-windows-latest/Windows_Server-%s-English-%s-EKS_Optimized-%s/image_id", w.Version, v1.WindowsCore, k8sVersion)) + if err != nil { + return DescribeImageQuery{}, fmt.Errorf(`failed to discover any AMIs for alias "windows%s@%s"`, w.Version, amiVersion) } + return DescribeImageQuery{ + Filters: []*ec2.Filter{&ec2.Filter{ + Name: lo.ToPtr("image-id"), + Values: []*string{lo.ToPtr(imageID)}, + }}, + KnownRequirements: map[string][]scheduling.Requirements{ + imageID: []scheduling.Requirements{scheduling.NewRequirements( + scheduling.NewRequirement(corev1.LabelOSStable, corev1.NodeSelectorOpIn, string(corev1.Windows)), + scheduling.NewRequirement(corev1.LabelWindowsBuild, corev1.NodeSelectorOpIn, w.Build), + )}, + }, + }, nil } // UserData returns the default userdata script for the AMI Family -func (w Windows) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ []*cloudprovider.InstanceType, customUserData *string, _ *v1beta1.InstanceStorePolicy) bootstrap.Bootstrapper { +func (w Windows) UserData(kubeletConfig *v1.KubeletConfiguration, taints []corev1.Taint, labels map[string]string, caBundle *string, _ []*cloudprovider.InstanceType, customUserData *string, _ *v1.InstanceStorePolicy) bootstrap.Bootstrapper { return bootstrap.Windows{ Options: bootstrap.Options{ ClusterName: w.Options.ClusterName, @@ -70,10 +81,10 @@ func (w Windows) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taint } // DefaultBlockDeviceMappings returns the default block device mappings for the AMI Family -func (w Windows) DefaultBlockDeviceMappings() []*v1beta1.BlockDeviceMapping { +func (w Windows) DefaultBlockDeviceMappings() []*v1.BlockDeviceMapping { sda1EBS := DefaultEBS sda1EBS.VolumeSize = lo.ToPtr(resource.MustParse("50Gi")) - return []*v1beta1.BlockDeviceMapping{{ + return []*v1.BlockDeviceMapping{{ DeviceName: w.EphemeralBlockDevice(), EBS: &sda1EBS, }} diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index 50abbaf45826..0fe9e783a17c 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -28,14 +28,14 @@ import ( "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/samber/lo" "go.uber.org/multierr" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" - "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/log" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/utils/resources" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/batcher" "github.com/aws/karpenter-provider-aws/pkg/cache" awserrors "github.com/aws/karpenter-provider-aws/pkg/errors" @@ -49,9 +49,12 @@ import ( "sigs.k8s.io/karpenter/pkg/scheduling" ) -var ( +const ( instanceTypeFlexibilityThreshold = 5 // falling back to on-demand without flexibility risks insufficient capacity errors + maxInstanceTypes = 60 +) +var ( instanceStateFilter = &ec2.Filter{ Name: aws.String("instance-state-name"), Values: aws.StringSlice([]string{ec2.InstanceStateNamePending, ec2.InstanceStateNameRunning, ec2.InstanceStateNameStopping, ec2.InstanceStateNameStopped, ec2.InstanceStateNameShuttingDown}), @@ -59,7 +62,7 @@ var ( ) type Provider interface { - Create(context.Context, *v1beta1.EC2NodeClass, *corev1beta1.NodeClaim, []*cloudprovider.InstanceType) (*Instance, error) + Create(context.Context, *v1.EC2NodeClass, *karpv1.NodeClaim, []*cloudprovider.InstanceType) (*Instance, error) Get(context.Context, string) (*Instance, error) List(context.Context) ([]*Instance, error) Delete(context.Context, string) error @@ -89,12 +92,16 @@ func NewDefaultProvider(ctx context.Context, region string, ec2api ec2iface.EC2A } } -func (p *DefaultProvider) Create(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) (*Instance, error) { +func (p *DefaultProvider) Create(ctx context.Context, nodeClass *v1.EC2NodeClass, nodeClaim *karpv1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) (*Instance, error) { schedulingRequirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...) // Only filter the instances if there are no minValues in the requirement. if !schedulingRequirements.HasMinValues() { instanceTypes = p.filterInstanceTypes(nodeClaim, instanceTypes) } + instanceTypes, err := cloudprovider.InstanceTypes(instanceTypes).Truncate(schedulingRequirements, maxInstanceTypes) + if err != nil { + return nil, fmt.Errorf("truncating instance types, %w", err) + } tags := getTags(ctx, nodeClass, nodeClaim) fleetInstance, err := p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes, tags) if awserrors.IsLaunchTemplateNotFound(err) { @@ -105,7 +112,7 @@ func (p *DefaultProvider) Create(ctx context.Context, nodeClass *v1beta1.EC2Node if err != nil { return nil, err } - efaEnabled := lo.Contains(lo.Keys(nodeClaim.Spec.Resources.Requests), v1beta1.ResourceEFA) + efaEnabled := lo.Contains(lo.Keys(nodeClaim.Spec.Resources.Requests), v1.ResourceEFA) return NewInstanceFromFleet(fleetInstance, tags, efaEnabled), nil } @@ -136,11 +143,11 @@ func (p *DefaultProvider) List(ctx context.Context) ([]*Instance, error) { Filters: []*ec2.Filter{ { Name: aws.String("tag-key"), - Values: aws.StringSlice([]string{corev1beta1.NodePoolLabelKey}), + Values: aws.StringSlice([]string{karpv1.NodePoolLabelKey}), }, { Name: aws.String("tag-key"), - Values: aws.StringSlice([]string{v1beta1.LabelNodeClass}), + Values: aws.StringSlice([]string{v1.LabelNodeClass}), }, { Name: aws.String("tag-key"), @@ -166,7 +173,7 @@ func (p *DefaultProvider) Delete(ctx context.Context, id string) error { if awserrors.IsNotFound(err) { return cloudprovider.NewNodeClaimNotFoundError(fmt.Errorf("instance already terminated")) } - if _, e := p.Get(ctx, id); err != nil { + if _, e := p.Get(ctx, id); e != nil { if cloudprovider.IsNodeClaimNotFoundError(e) { return e } @@ -193,7 +200,7 @@ func (p *DefaultProvider) CreateTags(ctx context.Context, id string, tags map[st return nil } -func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, tags map[string]string) (*ec2.CreateFleetInstance, error) { +func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1.EC2NodeClass, nodeClaim *karpv1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, tags map[string]string) (*ec2.CreateFleetInstance, error) { capacityType := p.getCapacityType(nodeClaim, instanceTypes) zonalSubnets, err := p.subnetProvider.ZonalSubnetsForLaunch(ctx, nodeClass, instanceTypes, capacityType) if err != nil { @@ -206,7 +213,7 @@ func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1 return nil, fmt.Errorf("getting launch template configs, %w", err) } if err := p.checkODFallback(nodeClaim, instanceTypes, launchTemplateConfigs); err != nil { - logging.FromContext(ctx).Warn(err.Error()) + log.FromContext(ctx).Error(err, "failed while checking on-demand fallback") } // Create fleet createFleetInput := &ec2.CreateFleetInput{ @@ -223,7 +230,7 @@ func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1 {ResourceType: aws.String(ec2.ResourceTypeFleet), Tags: utils.MergeTags(tags)}, }, } - if capacityType == corev1beta1.CapacityTypeSpot { + if capacityType == karpv1.CapacityTypeSpot { createFleetInput.SpotOptions = &ec2.SpotOptionsRequest{AllocationStrategy: aws.String(ec2.SpotAllocationStrategyPriceCapacityOptimized)} } else { createFleetInput.OnDemandOptions = &ec2.OnDemandOptionsRequest{AllocationStrategy: aws.String(ec2.FleetOnDemandAllocationStrategyLowestPrice)} @@ -251,19 +258,19 @@ func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1 return createFleetOutput.Instances[0], nil } -func getTags(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim) map[string]string { +func getTags(ctx context.Context, nodeClass *v1.EC2NodeClass, nodeClaim *karpv1.NodeClaim) map[string]string { staticTags := map[string]string{ fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName): "owned", - corev1beta1.NodePoolLabelKey: nodeClaim.Labels[corev1beta1.NodePoolLabelKey], - corev1beta1.ManagedByAnnotationKey: options.FromContext(ctx).ClusterName, - v1beta1.LabelNodeClass: nodeClass.Name, + karpv1.NodePoolLabelKey: nodeClaim.Labels[karpv1.NodePoolLabelKey], + v1.EKSClusterNameTagKey: options.FromContext(ctx).ClusterName, + v1.LabelNodeClass: nodeClass.Name, } return lo.Assign(nodeClass.Spec.Tags, staticTags) } -func (p *DefaultProvider) checkODFallback(nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest) error { +func (p *DefaultProvider) checkODFallback(nodeClaim *karpv1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest) error { // only evaluate for on-demand fallback if the capacity type for the request is OD and both OD and spot are allowed in requirements - if p.getCapacityType(nodeClaim, instanceTypes) != corev1beta1.CapacityTypeOnDemand || !scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...).Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeSpot) { + if p.getCapacityType(nodeClaim, instanceTypes) != karpv1.CapacityTypeOnDemand || !scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...).Get(karpv1.CapacityTypeLabelKey).Has(karpv1.CapacityTypeSpot) { return nil } @@ -284,16 +291,18 @@ func (p *DefaultProvider) checkODFallback(nodeClaim *corev1beta1.NodeClaim, inst return nil } -func (p *DefaultProvider) getLaunchTemplateConfigs(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, - instanceTypes []*cloudprovider.InstanceType, zonalSubnets map[string]*ec2.Subnet, capacityType string, tags map[string]string) ([]*ec2.FleetLaunchTemplateConfigRequest, error) { +func (p *DefaultProvider) getLaunchTemplateConfigs(ctx context.Context, nodeClass *v1.EC2NodeClass, nodeClaim *karpv1.NodeClaim, + instanceTypes []*cloudprovider.InstanceType, zonalSubnets map[string]*subnet.Subnet, capacityType string, tags map[string]string) ([]*ec2.FleetLaunchTemplateConfigRequest, error) { var launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest launchTemplates, err := p.launchTemplateProvider.EnsureAll(ctx, nodeClass, nodeClaim, instanceTypes, capacityType, tags) if err != nil { return nil, fmt.Errorf("getting launch templates, %w", err) } + requirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...) + requirements[karpv1.CapacityTypeLabelKey] = scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, capacityType) for _, launchTemplate := range launchTemplates { launchTemplateConfig := &ec2.FleetLaunchTemplateConfigRequest{ - Overrides: p.getOverrides(launchTemplate.InstanceTypes, zonalSubnets, scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...).Get(v1.LabelTopologyZone), capacityType, launchTemplate.ImageID), + Overrides: p.getOverrides(launchTemplate.InstanceTypes, zonalSubnets, requirements, launchTemplate.ImageID), LaunchTemplateSpecification: &ec2.FleetLaunchTemplateSpecificationRequest{ LaunchTemplateName: aws.String(launchTemplate.Name), Version: aws.String("$Latest"), @@ -311,7 +320,7 @@ func (p *DefaultProvider) getLaunchTemplateConfigs(ctx context.Context, nodeClas // getOverrides creates and returns launch template overrides for the cross product of InstanceTypes and subnets (with subnets being constrained by // zones and the offerings in InstanceTypes) -func (p *DefaultProvider) getOverrides(instanceTypes []*cloudprovider.InstanceType, zonalSubnets map[string]*ec2.Subnet, zones *scheduling.Requirement, capacityType string, image string) []*ec2.FleetLaunchTemplateOverridesRequest { +func (p *DefaultProvider) getOverrides(instanceTypes []*cloudprovider.InstanceType, zonalSubnets map[string]*subnet.Subnet, reqs scheduling.Requirements, image string) []*ec2.FleetLaunchTemplateOverridesRequest { // Unwrap all the offerings to a flat slice that includes a pointer // to the parent instance type name type offeringWithParentName struct { @@ -328,26 +337,22 @@ func (p *DefaultProvider) getOverrides(instanceTypes []*cloudprovider.InstanceTy }) unwrappedOfferings = append(unwrappedOfferings, ofs...) } - var overrides []*ec2.FleetLaunchTemplateOverridesRequest for _, offering := range unwrappedOfferings { - if capacityType != offering.CapacityType { - continue - } - if !zones.Has(offering.Zone) { + if reqs.Compatible(offering.Requirements, scheduling.AllowUndefinedWellKnownLabels) != nil { continue } - subnet, ok := zonalSubnets[offering.Zone] + subnet, ok := zonalSubnets[offering.Requirements.Get(corev1.LabelTopologyZone).Any()] if !ok { continue } overrides = append(overrides, &ec2.FleetLaunchTemplateOverridesRequest{ InstanceType: aws.String(offering.parentInstanceTypeName), - SubnetId: subnet.SubnetId, + SubnetId: lo.ToPtr(subnet.ID), ImageId: aws.String(image), // This is technically redundant, but is useful if we have to parse insufficient capacity errors from // CreateFleet so that we can figure out the zone rather than additional API calls to look up the subnet - AvailabilityZone: subnet.AvailabilityZone, + AvailabilityZone: lo.ToPtr(subnet.Zone), }) } return overrides @@ -364,24 +369,24 @@ func (p *DefaultProvider) updateUnavailableOfferingsCache(ctx context.Context, e // getCapacityType selects spot if both constraints are flexible and there is an // available offering. The AWS Cloud Provider defaults to [ on-demand ], so spot // must be explicitly included in capacity type requirements. -func (p *DefaultProvider) getCapacityType(nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) string { - requirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim. - Spec.Requirements...) - if requirements.Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeSpot) { +func (p *DefaultProvider) getCapacityType(nodeClaim *karpv1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) string { + requirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...) + if requirements.Get(karpv1.CapacityTypeLabelKey).Has(karpv1.CapacityTypeSpot) { + requirements[karpv1.CapacityTypeLabelKey] = scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, karpv1.CapacityTypeSpot) for _, instanceType := range instanceTypes { for _, offering := range instanceType.Offerings.Available() { - if requirements.Get(v1.LabelTopologyZone).Has(offering.Zone) && offering.CapacityType == corev1beta1.CapacityTypeSpot { - return corev1beta1.CapacityTypeSpot + if requirements.Compatible(offering.Requirements, scheduling.AllowUndefinedWellKnownLabels) == nil { + return karpv1.CapacityTypeSpot } } } } - return corev1beta1.CapacityTypeOnDemand + return karpv1.CapacityTypeOnDemand } // filterInstanceTypes is used to provide filtering on the list of potential instance types to further limit it to those // that make the most sense given our specific AWS cloudprovider. -func (p *DefaultProvider) filterInstanceTypes(nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) []*cloudprovider.InstanceType { +func (p *DefaultProvider) filterInstanceTypes(nodeClaim *karpv1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) []*cloudprovider.InstanceType { instanceTypes = filterExoticInstanceTypes(instanceTypes) // If we could potentially launch either a spot or on-demand node, we want to filter out the spot instance types that // are more expensive than the cheapest on-demand type. @@ -393,24 +398,25 @@ func (p *DefaultProvider) filterInstanceTypes(nodeClaim *corev1beta1.NodeClaim, // isMixedCapacityLaunch returns true if nodepools and available offerings could potentially allow either a spot or // and on-demand node to launch -func (p *DefaultProvider) isMixedCapacityLaunch(nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) bool { +func (p *DefaultProvider) isMixedCapacityLaunch(nodeClaim *karpv1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) bool { requirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...) // requirements must allow both - if !requirements.Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeSpot) || - !requirements.Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeOnDemand) { + if !requirements.Get(karpv1.CapacityTypeLabelKey).Has(karpv1.CapacityTypeSpot) || + !requirements.Get(karpv1.CapacityTypeLabelKey).Has(karpv1.CapacityTypeOnDemand) { return false } hasSpotOfferings := false hasODOffering := false - if requirements.Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeSpot) { + if requirements.Get(karpv1.CapacityTypeLabelKey).Has(karpv1.CapacityTypeSpot) { for _, instanceType := range instanceTypes { for _, offering := range instanceType.Offerings.Available() { - if requirements.Get(v1.LabelTopologyZone).Has(offering.Zone) { - if offering.CapacityType == corev1beta1.CapacityTypeSpot { - hasSpotOfferings = true - } else { - hasODOffering = true - } + if requirements.Compatible(offering.Requirements, scheduling.AllowUndefinedWellKnownLabels) != nil { + continue + } + if offering.Requirements.Get(karpv1.CapacityTypeLabelKey).Any() == karpv1.CapacityTypeSpot { + hasSpotOfferings = true + } else { + hasODOffering = true } } } @@ -425,7 +431,7 @@ func filterUnwantedSpot(instanceTypes []*cloudprovider.InstanceType) []*cloudpro // first, find the price of our cheapest available on-demand instance type that could support this node for _, it := range instanceTypes { for _, o := range it.Offerings.Available() { - if o.CapacityType == corev1beta1.CapacityTypeOnDemand && o.Price < cheapestOnDemand { + if o.Requirements.Get(karpv1.CapacityTypeLabelKey).Any() == karpv1.CapacityTypeOnDemand && o.Price < cheapestOnDemand { cheapestOnDemand = o.Price } } @@ -452,13 +458,13 @@ func filterExoticInstanceTypes(instanceTypes []*cloudprovider.InstanceType) []*c for _, it := range instanceTypes { // deprioritize metal even if our opinionated filter isn't applied due to something like an instance family // requirement - if _, ok := lo.Find(it.Requirements.Get(v1beta1.LabelInstanceSize).Values(), func(size string) bool { return strings.Contains(size, "metal") }); ok { + if _, ok := lo.Find(it.Requirements.Get(v1.LabelInstanceSize).Values(), func(size string) bool { return strings.Contains(size, "metal") }); ok { continue } - if !resources.IsZero(it.Capacity[v1beta1.ResourceAWSNeuron]) || - !resources.IsZero(it.Capacity[v1beta1.ResourceAMDGPU]) || - !resources.IsZero(it.Capacity[v1beta1.ResourceNVIDIAGPU]) || - !resources.IsZero(it.Capacity[v1beta1.ResourceHabanaGaudi]) { + if !resources.IsZero(it.Capacity[v1.ResourceAWSNeuron]) || + !resources.IsZero(it.Capacity[v1.ResourceAMDGPU]) || + !resources.IsZero(it.Capacity[v1.ResourceNVIDIAGPU]) || + !resources.IsZero(it.Capacity[v1.ResourceHabanaGaudi]) { continue } genericInstanceTypes = append(genericInstanceTypes, it) diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index 45d3504bafb6..a006d3db8f35 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -20,21 +20,23 @@ import ( "testing" "time" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/awslabs/operatorpkg/object" "github.com/samber/lo" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/events" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" @@ -43,8 +45,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) var ctx context.Context @@ -59,12 +61,12 @@ func TestAWS(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) awsEnv = test.NewEnvironment(ctx, env) cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), - env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider, awsEnv.SubnetProvider) + env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider) }) var _ = AfterSuite(func() { @@ -78,42 +80,51 @@ var _ = BeforeEach(func() { }) var _ = Describe("InstanceProvider", func() { - var nodeClass *v1beta1.EC2NodeClass - var nodePool *corev1beta1.NodePool - var nodeClaim *corev1beta1.NodeClaim + var nodeClass *v1.EC2NodeClass + var nodePool *karpv1.NodePool + var nodeClaim *karpv1.NodeClaim BeforeEach(func() { nodeClass = test.EC2NodeClass() - nodePool = coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }, }, }) - nodeClaim = coretest.NodeClaim(corev1beta1.NodeClaim{ + nodeClaim = coretest.NodeClaim(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - corev1beta1.NodePoolLabelKey: nodePool.Name, + karpv1.NodePoolLabelKey: nodePool.Name, }, }, - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) + _, err := awsEnv.SubnetProvider.List(ctx, nodeClass) // Hydrate the subnet cache + Expect(err).To(BeNil()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) }) It("should return an ICE error when all attempted instance types return an ICE error", func() { ExpectApplied(ctx, env.Client, nodeClaim, nodePool, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{ - {CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, - {CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, - {CapacityType: corev1beta1.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, - {CapacityType: corev1beta1.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, + {CapacityType: karpv1.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, + {CapacityType: karpv1.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, + {CapacityType: karpv1.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, + {CapacityType: karpv1.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, }) instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, nodePool) Expect(err).ToNot(HaveOccurred()) @@ -143,15 +154,15 @@ var _ = Describe("InstanceProvider", func() { Value: aws.String("owned"), }, { - Key: aws.String(corev1beta1.NodePoolLabelKey), + Key: aws.String(karpv1.NodePoolLabelKey), Value: aws.String("default"), }, { - Key: aws.String(v1beta1.LabelNodeClass), + Key: aws.String(v1.LabelNodeClass), Value: aws.String("default"), }, { - Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Key: aws.String(v1.EKSClusterNameTagKey), Value: aws.String(options.FromContext(ctx).ClusterName), }, }, diff --git a/pkg/providers/instance/types.go b/pkg/providers/instance/types.go index 5f3804f2d004..5ee8e3b3a930 100644 --- a/pkg/providers/instance/types.go +++ b/pkg/providers/instance/types.go @@ -21,7 +21,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/samber/lo" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" ) // Instance is an internal data representation of either an ec2.Instance or an ec2.FleetInstance @@ -48,7 +48,7 @@ func NewInstance(out *ec2.Instance) *Instance { ImageID: aws.StringValue(out.ImageId), Type: aws.StringValue(out.InstanceType), Zone: aws.StringValue(out.Placement.AvailabilityZone), - CapacityType: lo.Ternary(out.SpotInstanceRequestId != nil, corev1beta1.CapacityTypeSpot, corev1beta1.CapacityTypeOnDemand), + CapacityType: lo.Ternary(out.SpotInstanceRequestId != nil, karpv1.CapacityTypeSpot, karpv1.CapacityTypeOnDemand), SecurityGroupIDs: lo.Map(out.SecurityGroups, func(securitygroup *ec2.GroupIdentifier, _ int) string { return aws.StringValue(securitygroup.GroupId) }), diff --git a/pkg/providers/instanceprofile/instanceprofile.go b/pkg/providers/instanceprofile/instanceprofile.go index d8b361a8fd23..8dd66d95b7aa 100644 --- a/pkg/providers/instanceprofile/instanceprofile.go +++ b/pkg/providers/instanceprofile/instanceprofile.go @@ -23,9 +23,10 @@ import ( "github.com/aws/aws-sdk-go/service/iam/iamiface" "github.com/patrickmn/go-cache" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" awserrors "github.com/aws/karpenter-provider-aws/pkg/errors" "github.com/aws/karpenter-provider-aws/pkg/operator/options" ) @@ -59,7 +60,7 @@ func NewDefaultProvider(region string, iamapi iamiface.IAMAPI, cache *cache.Cach func (p *DefaultProvider) Create(ctx context.Context, m ResourceOwner) (string, error) { profileName := m.InstanceProfileName(options.FromContext(ctx).ClusterName, p.region) - tags := lo.Assign(m.InstanceProfileTags(options.FromContext(ctx).ClusterName), map[string]string{v1.LabelTopologyRegion: p.region}) + tags := lo.Assign(m.InstanceProfileTags(options.FromContext(ctx).ClusterName), map[string]string{corev1.LabelTopologyRegion: p.region}) // An instance profile exists for this NodeClass if _, ok := p.cache.Get(string(m.GetUID())); ok { @@ -81,6 +82,16 @@ func (p *DefaultProvider) Create(ctx context.Context, m ResourceOwner) (string, } instanceProfile = o.InstanceProfile } else { + if !lo.ContainsBy(out.InstanceProfile.Tags, func(t *iam.Tag) bool { + return lo.FromPtr(t.Key) == v1.EKSClusterNameTagKey + }) { + if _, err = p.iamapi.TagInstanceProfileWithContext(ctx, &iam.TagInstanceProfileInput{ + InstanceProfileName: aws.String(profileName), + Tags: lo.MapToSlice(tags, func(k, v string) *iam.Tag { return &iam.Tag{Key: aws.String(k), Value: aws.String(v)} }), + }); err != nil { + return "", fmt.Errorf("tagging instance profile %q, %w", profileName, err) + } + } instanceProfile = out.InstanceProfile } // Instance profiles can only have a single role assigned to them so this profile either has 1 or 0 roles diff --git a/pkg/providers/instancetype/instancetype.go b/pkg/providers/instancetype/instancetype.go index af3c82406711..3607782b4482 100644 --- a/pkg/providers/instancetype/instancetype.go +++ b/pkg/providers/instancetype/instancetype.go @@ -24,37 +24,34 @@ import ( "github.com/mitchellh/hashstructure/v2" "github.com/patrickmn/go-cache" "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/log" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + "sigs.k8s.io/karpenter/pkg/scheduling" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" awscache "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" - "knative.dev/pkg/logging" + "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" "github.com/aws/karpenter-provider-aws/pkg/providers/subnet" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/utils/pretty" - - "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" -) - -const ( - InstanceTypesCacheKey = "types" - InstanceTypeOfferingsCacheKey = "offerings" ) type Provider interface { LivenessProbe(*http.Request) error - - List(context.Context, *corev1beta1.KubeletConfiguration, *v1beta1.EC2NodeClass) ([]*cloudprovider.InstanceType, error) + List(context.Context, *v1.KubeletConfiguration, *v1.EC2NodeClass) ([]*cloudprovider.InstanceType, error) + UpdateInstanceTypes(ctx context.Context) error + UpdateInstanceTypeOfferings(ctx context.Context) error } type DefaultProvider struct { @@ -62,14 +59,19 @@ type DefaultProvider struct { ec2api ec2iface.EC2API subnetProvider subnet.Provider pricingProvider pricing.Provider - // Has one cache entry for all the instance types (key: InstanceTypesCacheKey) - // Has one cache entry for all the zones for each subnet selector (key: InstanceTypesZonesCacheKeyPrefix:) - // Values cached *before* considering insufficient capacity errors from the unavailableOfferings cache. + + // Values stored *before* considering insufficient capacity errors from the unavailableOfferings cache. // Fully initialized Instance Types are also cached based on the set of all instance types, zones, unavailableOfferings cache, // EC2NodeClass, and kubelet configuration from the NodePool - mu sync.Mutex - cache *cache.Cache + muInstanceTypeInfo sync.RWMutex + // TODO @engedaam: Look into only storing the needed EC2InstanceTypeInfo + instanceTypesInfo []*ec2.InstanceTypeInfo + + muInstanceTypeOfferings sync.RWMutex + instanceTypeOfferings map[string]sets.Set[string] + + instanceTypesCache *cache.Cache unavailableOfferings *awscache.UnavailableOfferings cm *pretty.ChangeMonitor @@ -79,45 +81,44 @@ type DefaultProvider struct { instanceTypeOfferingsSeqNum uint64 } -func NewDefaultProvider(region string, cache *cache.Cache, ec2api ec2iface.EC2API, subnetProvider subnet.Provider, +func NewDefaultProvider(region string, instanceTypesCache *cache.Cache, ec2api ec2iface.EC2API, subnetProvider subnet.Provider, unavailableOfferingsCache *awscache.UnavailableOfferings, pricingProvider pricing.Provider) *DefaultProvider { return &DefaultProvider{ - ec2api: ec2api, - region: region, - subnetProvider: subnetProvider, - pricingProvider: pricingProvider, - cache: cache, - unavailableOfferings: unavailableOfferingsCache, - cm: pretty.NewChangeMonitor(), - instanceTypesSeqNum: 0, + ec2api: ec2api, + region: region, + subnetProvider: subnetProvider, + pricingProvider: pricingProvider, + instanceTypesInfo: []*ec2.InstanceTypeInfo{}, + instanceTypeOfferings: map[string]sets.Set[string]{}, + instanceTypesCache: instanceTypesCache, + unavailableOfferings: unavailableOfferingsCache, + cm: pretty.NewChangeMonitor(), + instanceTypesSeqNum: 0, } } -func (p *DefaultProvider) List(ctx context.Context, kc *corev1beta1.KubeletConfiguration, nodeClass *v1beta1.EC2NodeClass) ([]*cloudprovider.InstanceType, error) { - // Get InstanceTypes from EC2 - instanceTypes, err := p.GetInstanceTypes(ctx) - if err != nil { - return nil, err - } - // Get InstanceTypeOfferings from EC2 - instanceTypeOfferings, err := p.getInstanceTypeOfferings(ctx) - if err != nil { - return nil, err - } - subnets, err := p.subnetProvider.List(ctx, nodeClass) - if err != nil { - return nil, err - } - subnetZones := sets.New[string](lo.Map(subnets, func(s *ec2.Subnet, _ int) string { - return aws.StringValue(s.AvailabilityZone) - })...) +func (p *DefaultProvider) List(ctx context.Context, kc *v1.KubeletConfiguration, nodeClass *v1.EC2NodeClass) ([]*cloudprovider.InstanceType, error) { + p.muInstanceTypeInfo.RLock() + p.muInstanceTypeOfferings.RLock() + defer p.muInstanceTypeInfo.RUnlock() + defer p.muInstanceTypeOfferings.RUnlock() if kc == nil { - kc = &corev1beta1.KubeletConfiguration{} + kc = &v1.KubeletConfiguration{} } - if nodeClass == nil { - nodeClass = &v1beta1.EC2NodeClass{} + if len(p.instanceTypesInfo) == 0 { + return nil, fmt.Errorf("no instance types found") } + if len(p.instanceTypeOfferings) == 0 { + return nil, fmt.Errorf("no instance types offerings found") + } + if len(nodeClass.Status.Subnets) == 0 { + return nil, fmt.Errorf("no subnets found") + } + + subnetZones := sets.New(lo.Map(nodeClass.Status.Subnets, func(s v1.Subnet, _ int) string { + return aws.StringValue(&s.Zone) + })...) // Compute fully initialized instance types hash key subnetZonesHash, _ := hashstructure.Hash(subnetZones, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}) @@ -130,26 +131,28 @@ func (p *DefaultProvider) List(ctx context.Context, kc *corev1beta1.KubeletConfi subnetZonesHash, kcHash, blockDeviceMappingsHash, - aws.StringValue((*string)(nodeClass.Spec.InstanceStorePolicy)), - aws.StringValue(nodeClass.Spec.AMIFamily), + lo.FromPtr((*string)(nodeClass.Spec.InstanceStorePolicy)), + nodeClass.AMIFamily(), ) - if item, ok := p.cache.Get(key); ok { - return item.([]*cloudprovider.InstanceType), nil + if item, ok := p.instanceTypesCache.Get(key); ok { + // Ensure what's returned from this function is a shallow-copy of the slice (not a deep-copy of the data itself) + // so that modifications to the ordering of the data don't affect the original + return append([]*cloudprovider.InstanceType{}, item.([]*cloudprovider.InstanceType)...), nil } // Get all zones across all offerings // We don't use this in the cache key since this is produced from our instanceTypeOfferings which we do cache allZones := sets.New[string]() - for _, offeringZones := range instanceTypeOfferings { + for _, offeringZones := range p.instanceTypeOfferings { for zone := range offeringZones { allZones.Insert(zone) } } if p.cm.HasChanged("zones", allZones) { - logging.FromContext(ctx).With("zones", allZones.UnsortedList()).Debugf("discovered zones") + log.FromContext(ctx).WithValues("zones", allZones.UnsortedList()).V(1).Info("discovered zones") } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) - result := lo.Map(instanceTypes, func(i *ec2.InstanceTypeInfo, _ int) *cloudprovider.InstanceType { + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) + result := lo.Map(p.instanceTypesInfo, func(i *ec2.InstanceTypeInfo, _ int) *cloudprovider.InstanceType { instanceTypeVCPU.With(prometheus.Labels{ instanceTypeLabel: *i.InstanceType, }).Set(float64(aws.Int64Value(i.VCpuInfo.DefaultVCpus))) @@ -164,9 +167,10 @@ func (p *DefaultProvider) List(ctx context.Context, kc *corev1beta1.KubeletConfi return NewInstanceType(ctx, i, p.region, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, kc.MaxPods, kc.PodsPerCore, kc.KubeReserved, kc.SystemReserved, kc.EvictionHard, kc.EvictionSoft, - amiFamily, p.createOfferings(ctx, i, instanceTypeOfferings[aws.StringValue(i.InstanceType)], allZones, subnetZones)) + amiFamily, p.createOfferings(ctx, i, allZones, p.instanceTypeOfferings[aws.StringValue(i.InstanceType)], nodeClass.Status.Subnets), + ) }) - p.cache.SetDefault(key, result) + p.instanceTypesCache.SetDefault(key, result) return result, nil } @@ -177,7 +181,87 @@ func (p *DefaultProvider) LivenessProbe(req *http.Request) error { return p.pricingProvider.LivenessProbe(req) } -func (p *DefaultProvider) createOfferings(ctx context.Context, instanceType *ec2.InstanceTypeInfo, instanceTypeZones, zones, subnetZones sets.Set[string]) []cloudprovider.Offering { +func (p *DefaultProvider) UpdateInstanceTypes(ctx context.Context) error { + // DO NOT REMOVE THIS LOCK ---------------------------------------------------------------------------- + // We lock here so that multiple callers to getInstanceTypeOfferings do not result in cache misses and multiple + // calls to EC2 when we could have just made one call. + // TODO @joinnis: This can be made more efficient by holding a Read lock and only obtaining the Write if not in cache + p.muInstanceTypeInfo.Lock() + defer p.muInstanceTypeInfo.Unlock() + var instanceTypes []*ec2.InstanceTypeInfo + + if err := p.ec2api.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("supported-virtualization-type"), + Values: []*string{aws.String("hvm")}, + }, + { + Name: aws.String("processor-info.supported-architecture"), + Values: aws.StringSlice([]string{"x86_64", "arm64"}), + }, + }, + }, func(page *ec2.DescribeInstanceTypesOutput, lastPage bool) bool { + instanceTypes = append(instanceTypes, page.InstanceTypes...) + return true + }); err != nil { + return fmt.Errorf("describing instance types, %w", err) + } + + if p.cm.HasChanged("instance-types", instanceTypes) { + // Only update instanceTypesSeqNun with the instance types have been changed + // This is to not create new keys with duplicate instance types option + atomic.AddUint64(&p.instanceTypesSeqNum, 1) + log.FromContext(ctx).WithValues( + "count", len(instanceTypes)).V(1).Info("discovered instance types") + } + p.instanceTypesInfo = instanceTypes + return nil +} + +func (p *DefaultProvider) UpdateInstanceTypeOfferings(ctx context.Context) error { + // DO NOT REMOVE THIS LOCK ---------------------------------------------------------------------------- + // We lock here so that multiple callers to GetInstanceTypes do not result in cache misses and multiple + // calls to EC2 when we could have just made one call. This lock is here because multiple callers to EC2 result + // in A LOT of extra memory generated from the response for simultaneous callers. + // TODO @joinnis: This can be made more efficient by holding a Read lock and only obtaining the Write if not in cache + p.muInstanceTypeOfferings.Lock() + defer p.muInstanceTypeOfferings.Unlock() + + // Get offerings from EC2 + instanceTypeOfferings := map[string]sets.Set[string]{} + if err := p.ec2api.DescribeInstanceTypeOfferingsPagesWithContext(ctx, &ec2.DescribeInstanceTypeOfferingsInput{LocationType: aws.String("availability-zone")}, + func(output *ec2.DescribeInstanceTypeOfferingsOutput, lastPage bool) bool { + for _, offering := range output.InstanceTypeOfferings { + if _, ok := instanceTypeOfferings[aws.StringValue(offering.InstanceType)]; !ok { + instanceTypeOfferings[aws.StringValue(offering.InstanceType)] = sets.New[string]() + } + instanceTypeOfferings[aws.StringValue(offering.InstanceType)].Insert(aws.StringValue(offering.Location)) + } + return true + }); err != nil { + return fmt.Errorf("describing instance type zone offerings, %w", err) + } + if p.cm.HasChanged("instance-type-offering", instanceTypeOfferings) { + // Only update instanceTypesSeqNun with the instance type offerings have been changed + // This is to not create new keys with duplicate instance type offerings option + atomic.AddUint64(&p.instanceTypeOfferingsSeqNum, 1) + log.FromContext(ctx).WithValues("instance-type-count", len(instanceTypeOfferings)).V(1).Info("discovered offerings for instance types") + } + p.instanceTypeOfferings = instanceTypeOfferings + return nil +} + +// createOfferings creates a set of mutually exclusive offerings for a given instance type. This provider maintains an +// invariant that each offering is mutually exclusive. Specifically, there is an offering for each permutation of zone +// and capacity type. ZoneID is also injected into the offering requirements, when available, but there is a 1-1 +// mapping between zone and zoneID so this does not change the number of offerings. +// +// Each requirement on the offering is guaranteed to have a single value. To get the value for a requirement on an +// offering, you can do the following thanks to this invariant: +// +// offering.Requirements.Get(v1.TopologyLabelZone).Any() +func (p *DefaultProvider) createOfferings(ctx context.Context, instanceType *ec2.InstanceTypeInfo, zones, instanceTypeZones sets.Set[string], subnets []v1.Subnet) []cloudprovider.Offering { var offerings []cloudprovider.Offering for zone := range zones { // while usage classes should be a distinct set, there's no guarantee of that @@ -195,16 +279,26 @@ func (p *DefaultProvider) createOfferings(ctx context.Context, instanceType *ec2 // ignore since karpenter doesn't support it yet, but do not log an unknown capacity type error continue default: - logging.FromContext(ctx).Errorf("Received unknown capacity type %s for instance type %s", capacityType, *instanceType.InstanceType) + log.FromContext(ctx).WithValues("capacity-type", capacityType, "instance-type", *instanceType.InstanceType).Error(fmt.Errorf("received unknown capacity type"), "failed parsing offering") continue } - available := !isUnavailable && ok && instanceTypeZones.Has(zone) && subnetZones.Has(zone) - offerings = append(offerings, cloudprovider.Offering{ - Zone: zone, - CapacityType: capacityType, - Price: price, - Available: available, + + subnet, hasSubnet := lo.Find(subnets, func(s v1.Subnet) bool { + return s.Zone == zone }) + available := !isUnavailable && ok && instanceTypeZones.Has(zone) && hasSubnet + offering := cloudprovider.Offering{ + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, capacityType), + scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, zone), + ), + Price: price, + Available: available, + } + if subnet.ZoneID != "" { + offering.Requirements.Add(scheduling.NewRequirement(v1.LabelTopologyZoneID, corev1.NodeSelectorOpIn, subnet.ZoneID)) + } + offerings = append(offerings, offering) instanceTypeOfferingAvailable.With(prometheus.Labels{ instanceTypeLabel: *instanceType.InstanceType, capacityTypeLabel: capacityType, @@ -220,79 +314,8 @@ func (p *DefaultProvider) createOfferings(ctx context.Context, instanceType *ec2 return offerings } -func (p *DefaultProvider) getInstanceTypeOfferings(ctx context.Context) (map[string]sets.Set[string], error) { - // DO NOT REMOVE THIS LOCK ---------------------------------------------------------------------------- - // We lock here so that multiple callers to getInstanceTypeOfferings do not result in cache misses and multiple - // calls to EC2 when we could have just made one call. - // TODO @joinnis: This can be made more efficient by holding a Read lock and only obtaining the Write if not in cache - p.mu.Lock() - defer p.mu.Unlock() - if cached, ok := p.cache.Get(InstanceTypeOfferingsCacheKey); ok { - return cached.(map[string]sets.Set[string]), nil - } - - // Get offerings from EC2 - instanceTypeOfferings := map[string]sets.Set[string]{} - if err := p.ec2api.DescribeInstanceTypeOfferingsPagesWithContext(ctx, &ec2.DescribeInstanceTypeOfferingsInput{LocationType: aws.String("availability-zone")}, - func(output *ec2.DescribeInstanceTypeOfferingsOutput, lastPage bool) bool { - for _, offering := range output.InstanceTypeOfferings { - if _, ok := instanceTypeOfferings[aws.StringValue(offering.InstanceType)]; !ok { - instanceTypeOfferings[aws.StringValue(offering.InstanceType)] = sets.New[string]() - } - instanceTypeOfferings[aws.StringValue(offering.InstanceType)].Insert(aws.StringValue(offering.Location)) - } - return true - }); err != nil { - return nil, fmt.Errorf("describing instance type zone offerings, %w", err) - } - if p.cm.HasChanged("instance-type-offering", instanceTypeOfferings) { - // Only update instanceTypesSeqNun with the instance type offerings have been changed - // This is to not create new keys with duplicate instance type offerings option - atomic.AddUint64(&p.instanceTypeOfferingsSeqNum, 1) - logging.FromContext(ctx).With("instance-type-count", len(instanceTypeOfferings)).Debugf("discovered offerings for instance types") - } - p.cache.SetDefault(InstanceTypeOfferingsCacheKey, instanceTypeOfferings) - return instanceTypeOfferings, nil -} - -// GetInstanceTypes retrieves all instance types from the ec2 DescribeInstanceTypes API using some opinionated filters -func (p *DefaultProvider) GetInstanceTypes(ctx context.Context) ([]*ec2.InstanceTypeInfo, error) { - // DO NOT REMOVE THIS LOCK ---------------------------------------------------------------------------- - // We lock here so that multiple callers to GetInstanceTypes do not result in cache misses and multiple - // calls to EC2 when we could have just made one call. This lock is here because multiple callers to EC2 result - // in A LOT of extra memory generated from the response for simultaneous callers. - // TODO @joinnis: This can be made more efficient by holding a Read lock and only obtaining the Write if not in cache - p.mu.Lock() - defer p.mu.Unlock() - - if cached, ok := p.cache.Get(InstanceTypesCacheKey); ok { - return cached.([]*ec2.InstanceTypeInfo), nil - } - var instanceTypes []*ec2.InstanceTypeInfo - if err := p.ec2api.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{ - Filters: []*ec2.Filter{ - { - Name: aws.String("supported-virtualization-type"), - Values: []*string{aws.String("hvm")}, - }, - { - Name: aws.String("processor-info.supported-architecture"), - Values: aws.StringSlice([]string{"x86_64", "arm64"}), - }, - }, - }, func(page *ec2.DescribeInstanceTypesOutput, lastPage bool) bool { - instanceTypes = append(instanceTypes, page.InstanceTypes...) - return true - }); err != nil { - return nil, fmt.Errorf("fetching instance types using ec2.DescribeInstanceTypes, %w", err) - } - if p.cm.HasChanged("instance-types", instanceTypes) { - // Only update instanceTypesSeqNun with the instance types have been changed - // This is to not create new keys with duplicate instance types option - atomic.AddUint64(&p.instanceTypesSeqNum, 1) - logging.FromContext(ctx).With( - "count", len(instanceTypes)).Debugf("discovered instance types") - } - p.cache.SetDefault(InstanceTypesCacheKey, instanceTypes) - return instanceTypes, nil +func (p *DefaultProvider) Reset() { + p.instanceTypesInfo = []*ec2.InstanceTypeInfo{} + p.instanceTypeOfferings = map[string]sets.Set[string]{} + p.instanceTypesCache.Flush() } diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 6b0d284d725b..4c2ff001b7bf 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -22,37 +22,42 @@ import ( "reflect" "sort" "strings" + "sync" "testing" "time" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/awslabs/operatorpkg/status" "github.com/imdario/mergo" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" - . "knative.dev/pkg/logging/testing" - "knative.dev/pkg/ptr" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/controllers/provisioning" "sigs.k8s.io/karpenter/pkg/controllers/state" "sigs.k8s.io/karpenter/pkg/events" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" "sigs.k8s.io/karpenter/pkg/scheduling" coretest "sigs.k8s.io/karpenter/pkg/test" + . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" + "sigs.k8s.io/karpenter/pkg/utils/resources" "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" @@ -76,14 +81,14 @@ func TestAWS(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) awsEnv = test.NewEnvironment(ctx, env) fakeClock = &clock.FakeClock{} cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), - env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider, awsEnv.SubnetProvider) - cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) + env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider) + cluster = state.NewCluster(fakeClock, env.Client) prov = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) }) @@ -105,56 +110,109 @@ var _ = AfterEach(func() { }) var _ = Describe("InstanceTypeProvider", func() { - var nodeClass, windowsNodeClass *v1beta1.EC2NodeClass - var nodePool, windowsNodePool *corev1beta1.NodePool + var nodeClass, windowsNodeClass *v1.EC2NodeClass + var nodePool, windowsNodePool *karpv1.NodePool BeforeEach(func() { - nodeClass = test.EC2NodeClass() - nodePool = coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodeClass = test.EC2NodeClass( + v1.EC2NodeClass{ + Status: v1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + SecurityGroups: []v1.SecurityGroup{ + { + ID: "sg-test1", + }, + { + ID: "sg-test2", + }, + { + ID: "sg-test3", + }, + }, + Subnets: []v1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + }, + }, + }, + ) + nodeClass.StatusConditions().SetTrue(status.ConditionReady) + nodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, }, }, }, - Kubelet: &corev1beta1.KubeletConfiguration{}, - NodeClassRef: &corev1beta1.NodeClassReference{ + NodeClassRef: &karpv1.NodeClassReference{ Name: nodeClass.Name, }, }, }, }, }) - windowsNodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ - AMIFamily: &v1beta1.AMIFamilyWindows2022, + windowsNodeClass = test.EC2NodeClass(v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: []v1.AMISelectorTerm{{ + Alias: "windows2022@latest", + }}, + }, + Status: v1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + SecurityGroups: nodeClass.Status.SecurityGroups, + Subnets: nodeClass.Status.Subnets, + AMIs: []v1.AMI{ + { + ID: "ami-window-test1", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureAmd64}}, + {Key: corev1.LabelOSStable, Operator: corev1.NodeSelectorOpIn, Values: []string{string(corev1.Windows)}}, + {Key: corev1.LabelWindowsBuild, Operator: corev1.NodeSelectorOpIn, Values: []string{v1.Windows2022Build}}, + }, + }, + }, }, }) - windowsNodePool = coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + windowsNodeClass.StatusConditions().SetTrue(status.ConditionReady) + windowsNodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, }, }, }, - NodeClassRef: &corev1beta1.NodeClassReference{ + NodeClassRef: &karpv1.NodeClassReference{ Name: windowsNodeClass.Name, }, }, }, }, }) + _, err := awsEnv.SubnetProvider.List(ctx, nodeClass) // Hydrate the subnet cache + Expect(err).To(BeNil()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) }) It("should support individual instance type labels", func() { @@ -162,46 +220,48 @@ var _ = Describe("InstanceTypeProvider", func() { nodeSelector := map[string]string{ // Well known - corev1beta1.NodePoolLabelKey: nodePool.Name, - v1.LabelTopologyRegion: fake.DefaultRegion, - v1.LabelTopologyZone: "test-zone-1a", - v1.LabelInstanceTypeStable: "g4dn.8xlarge", - v1.LabelOSStable: "linux", - v1.LabelArchStable: "amd64", - corev1beta1.CapacityTypeLabelKey: "on-demand", + karpv1.NodePoolLabelKey: nodePool.Name, + corev1.LabelTopologyRegion: fake.DefaultRegion, + corev1.LabelTopologyZone: "test-zone-1a", + corev1.LabelInstanceTypeStable: "g4dn.8xlarge", + corev1.LabelOSStable: "linux", + corev1.LabelArchStable: "amd64", + karpv1.CapacityTypeLabelKey: "on-demand", // Well Known to AWS - v1beta1.LabelInstanceHypervisor: "nitro", - v1beta1.LabelInstanceEncryptionInTransitSupported: "true", - v1beta1.LabelInstanceCategory: "g", - v1beta1.LabelInstanceGeneration: "4", - v1beta1.LabelInstanceFamily: "g4dn", - v1beta1.LabelInstanceSize: "8xlarge", - v1beta1.LabelInstanceCPU: "32", - v1beta1.LabelInstanceCPUManufacturer: "intel", - v1beta1.LabelInstanceMemory: "131072", - v1beta1.LabelInstanceNetworkBandwidth: "50000", - v1beta1.LabelInstanceGPUName: "t4", - v1beta1.LabelInstanceGPUManufacturer: "nvidia", - v1beta1.LabelInstanceGPUCount: "1", - v1beta1.LabelInstanceGPUMemory: "16384", - v1beta1.LabelInstanceLocalNVME: "900", - v1beta1.LabelInstanceAcceleratorName: "inferentia", - v1beta1.LabelInstanceAcceleratorManufacturer: "aws", - v1beta1.LabelInstanceAcceleratorCount: "1", + v1.LabelInstanceHypervisor: "nitro", + v1.LabelInstanceEncryptionInTransitSupported: "true", + v1.LabelInstanceCategory: "g", + v1.LabelInstanceGeneration: "4", + v1.LabelInstanceFamily: "g4dn", + v1.LabelInstanceSize: "8xlarge", + v1.LabelInstanceCPU: "32", + v1.LabelInstanceCPUManufacturer: "intel", + v1.LabelInstanceMemory: "131072", + v1.LabelInstanceEBSBandwidth: "9500", + v1.LabelInstanceNetworkBandwidth: "50000", + v1.LabelInstanceGPUName: "t4", + v1.LabelInstanceGPUManufacturer: "nvidia", + v1.LabelInstanceGPUCount: "1", + v1.LabelInstanceGPUMemory: "16384", + v1.LabelInstanceLocalNVME: "900", + v1.LabelInstanceAcceleratorName: "inferentia", + v1.LabelInstanceAcceleratorManufacturer: "aws", + v1.LabelInstanceAcceleratorCount: "1", + v1.LabelTopologyZoneID: "tstz1-1a", // Deprecated Labels - v1.LabelFailureDomainBetaRegion: fake.DefaultRegion, - v1.LabelFailureDomainBetaZone: "test-zone-1a", - "beta.kubernetes.io/arch": "amd64", - "beta.kubernetes.io/os": "linux", - v1.LabelInstanceType: "g4dn.8xlarge", - "topology.ebs.csi.aws.com/zone": "test-zone-1a", - v1.LabelWindowsBuild: v1beta1.Windows2022Build, + corev1.LabelFailureDomainBetaRegion: fake.DefaultRegion, + corev1.LabelFailureDomainBetaZone: "test-zone-1a", + "beta.kubernetes.io/arch": "amd64", + "beta.kubernetes.io/os": "linux", + corev1.LabelInstanceType: "g4dn.8xlarge", + "topology.ebs.csi.aws.com/zone": "test-zone-1a", + corev1.LabelWindowsBuild: v1.Windows2022Build, } // Ensure that we're exercising all well known labels - Expect(lo.Keys(nodeSelector)).To(ContainElements(append(corev1beta1.WellKnownLabels.UnsortedList(), lo.Keys(corev1beta1.NormalizedLabels)...))) + Expect(lo.Keys(nodeSelector)).To(ContainElements(append(karpv1.WellKnownLabels.UnsortedList(), lo.Keys(karpv1.NormalizedLabels)...))) - var pods []*v1.Pod + var pods []*corev1.Pod for key, value := range nodeSelector { pods = append(pods, coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{key: value}})) } @@ -215,47 +275,49 @@ var _ = Describe("InstanceTypeProvider", func() { nodeSelector := map[string]string{ // Well known - corev1beta1.NodePoolLabelKey: nodePool.Name, - v1.LabelTopologyRegion: fake.DefaultRegion, - v1.LabelTopologyZone: "test-zone-1a", - v1.LabelInstanceTypeStable: "g4dn.8xlarge", - v1.LabelOSStable: "linux", - v1.LabelArchStable: "amd64", - corev1beta1.CapacityTypeLabelKey: "on-demand", + karpv1.NodePoolLabelKey: nodePool.Name, + corev1.LabelTopologyRegion: fake.DefaultRegion, + corev1.LabelTopologyZone: "test-zone-1a", + corev1.LabelInstanceTypeStable: "g4dn.8xlarge", + corev1.LabelOSStable: "linux", + corev1.LabelArchStable: "amd64", + karpv1.CapacityTypeLabelKey: "on-demand", // Well Known to AWS - v1beta1.LabelInstanceHypervisor: "nitro", - v1beta1.LabelInstanceEncryptionInTransitSupported: "true", - v1beta1.LabelInstanceCategory: "g", - v1beta1.LabelInstanceGeneration: "4", - v1beta1.LabelInstanceFamily: "g4dn", - v1beta1.LabelInstanceSize: "8xlarge", - v1beta1.LabelInstanceCPU: "32", - v1beta1.LabelInstanceCPUManufacturer: "intel", - v1beta1.LabelInstanceMemory: "131072", - v1beta1.LabelInstanceNetworkBandwidth: "50000", - v1beta1.LabelInstanceGPUName: "t4", - v1beta1.LabelInstanceGPUManufacturer: "nvidia", - v1beta1.LabelInstanceGPUCount: "1", - v1beta1.LabelInstanceGPUMemory: "16384", - v1beta1.LabelInstanceLocalNVME: "900", + v1.LabelInstanceHypervisor: "nitro", + v1.LabelInstanceEncryptionInTransitSupported: "true", + v1.LabelInstanceCategory: "g", + v1.LabelInstanceGeneration: "4", + v1.LabelInstanceFamily: "g4dn", + v1.LabelInstanceSize: "8xlarge", + v1.LabelInstanceCPU: "32", + v1.LabelInstanceCPUManufacturer: "intel", + v1.LabelInstanceMemory: "131072", + v1.LabelInstanceEBSBandwidth: "9500", + v1.LabelInstanceNetworkBandwidth: "50000", + v1.LabelInstanceGPUName: "t4", + v1.LabelInstanceGPUManufacturer: "nvidia", + v1.LabelInstanceGPUCount: "1", + v1.LabelInstanceGPUMemory: "16384", + v1.LabelInstanceLocalNVME: "900", + v1.LabelTopologyZoneID: "tstz1-1a", // Deprecated Labels - v1.LabelFailureDomainBetaRegion: fake.DefaultRegion, - v1.LabelFailureDomainBetaZone: "test-zone-1a", - "beta.kubernetes.io/arch": "amd64", - "beta.kubernetes.io/os": "linux", - v1.LabelInstanceType: "g4dn.8xlarge", - "topology.ebs.csi.aws.com/zone": "test-zone-1a", + corev1.LabelFailureDomainBetaRegion: fake.DefaultRegion, + corev1.LabelFailureDomainBetaZone: "test-zone-1a", + "beta.kubernetes.io/arch": "amd64", + "beta.kubernetes.io/os": "linux", + corev1.LabelInstanceType: "g4dn.8xlarge", + "topology.ebs.csi.aws.com/zone": "test-zone-1a", } // Ensure that we're exercising all well known labels except for accelerator labels Expect(lo.Keys(nodeSelector)).To(ContainElements( append( - corev1beta1.WellKnownLabels.Difference(sets.New( - v1beta1.LabelInstanceAcceleratorCount, - v1beta1.LabelInstanceAcceleratorName, - v1beta1.LabelInstanceAcceleratorManufacturer, - v1.LabelWindowsBuild, - )).UnsortedList(), lo.Keys(corev1beta1.NormalizedLabels)...))) + karpv1.WellKnownLabels.Difference(sets.New( + v1.LabelInstanceAcceleratorCount, + v1.LabelInstanceAcceleratorName, + v1.LabelInstanceAcceleratorManufacturer, + corev1.LabelWindowsBuild, + )).UnsortedList(), lo.Keys(karpv1.NormalizedLabels)...))) pod := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: nodeSelector}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -266,45 +328,47 @@ var _ = Describe("InstanceTypeProvider", func() { nodeSelector := map[string]string{ // Well known - corev1beta1.NodePoolLabelKey: nodePool.Name, - v1.LabelTopologyRegion: fake.DefaultRegion, - v1.LabelTopologyZone: "test-zone-1a", - v1.LabelInstanceTypeStable: "inf1.2xlarge", - v1.LabelOSStable: "linux", - v1.LabelArchStable: "amd64", - corev1beta1.CapacityTypeLabelKey: "on-demand", + karpv1.NodePoolLabelKey: nodePool.Name, + corev1.LabelTopologyRegion: fake.DefaultRegion, + corev1.LabelTopologyZone: "test-zone-1a", + corev1.LabelInstanceTypeStable: "inf1.2xlarge", + corev1.LabelOSStable: "linux", + corev1.LabelArchStable: "amd64", + karpv1.CapacityTypeLabelKey: "on-demand", // Well Known to AWS - v1beta1.LabelInstanceHypervisor: "nitro", - v1beta1.LabelInstanceEncryptionInTransitSupported: "true", - v1beta1.LabelInstanceCategory: "inf", - v1beta1.LabelInstanceGeneration: "1", - v1beta1.LabelInstanceFamily: "inf1", - v1beta1.LabelInstanceSize: "2xlarge", - v1beta1.LabelInstanceCPU: "8", - v1beta1.LabelInstanceCPUManufacturer: "intel", - v1beta1.LabelInstanceMemory: "16384", - v1beta1.LabelInstanceNetworkBandwidth: "5000", - v1beta1.LabelInstanceAcceleratorName: "inferentia", - v1beta1.LabelInstanceAcceleratorManufacturer: "aws", - v1beta1.LabelInstanceAcceleratorCount: "1", + v1.LabelInstanceHypervisor: "nitro", + v1.LabelInstanceEncryptionInTransitSupported: "true", + v1.LabelInstanceCategory: "inf", + v1.LabelInstanceGeneration: "1", + v1.LabelInstanceFamily: "inf1", + v1.LabelInstanceSize: "2xlarge", + v1.LabelInstanceCPU: "8", + v1.LabelInstanceCPUManufacturer: "intel", + v1.LabelInstanceMemory: "16384", + v1.LabelInstanceEBSBandwidth: "4750", + v1.LabelInstanceNetworkBandwidth: "5000", + v1.LabelInstanceAcceleratorName: "inferentia", + v1.LabelInstanceAcceleratorManufacturer: "aws", + v1.LabelInstanceAcceleratorCount: "1", + v1.LabelTopologyZoneID: "tstz1-1a", // Deprecated Labels - v1.LabelFailureDomainBetaRegion: fake.DefaultRegion, - v1.LabelFailureDomainBetaZone: "test-zone-1a", - "beta.kubernetes.io/arch": "amd64", - "beta.kubernetes.io/os": "linux", - v1.LabelInstanceType: "inf1.2xlarge", - "topology.ebs.csi.aws.com/zone": "test-zone-1a", + corev1.LabelFailureDomainBetaRegion: fake.DefaultRegion, + corev1.LabelFailureDomainBetaZone: "test-zone-1a", + "beta.kubernetes.io/arch": "amd64", + "beta.kubernetes.io/os": "linux", + corev1.LabelInstanceType: "inf1.2xlarge", + "topology.ebs.csi.aws.com/zone": "test-zone-1a", } // Ensure that we're exercising all well known labels except for gpu labels and nvme - expectedLabels := append(corev1beta1.WellKnownLabels.Difference(sets.New( - v1beta1.LabelInstanceGPUCount, - v1beta1.LabelInstanceGPUName, - v1beta1.LabelInstanceGPUManufacturer, - v1beta1.LabelInstanceGPUMemory, - v1beta1.LabelInstanceLocalNVME, - v1.LabelWindowsBuild, - )).UnsortedList(), lo.Keys(corev1beta1.NormalizedLabels)...) + expectedLabels := append(karpv1.WellKnownLabels.Difference(sets.New( + v1.LabelInstanceGPUCount, + v1.LabelInstanceGPUName, + v1.LabelInstanceGPUManufacturer, + v1.LabelInstanceGPUMemory, + v1.LabelInstanceLocalNVME, + corev1.LabelWindowsBuild, + )).UnsortedList(), lo.Keys(karpv1.NormalizedLabels)...) Expect(lo.Keys(nodeSelector)).To(ContainElements(expectedLabels)) pod := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: nodeSelector}) @@ -315,11 +379,11 @@ var _ = Describe("InstanceTypeProvider", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{ - v1.LabelInstanceTypeStable: "t3.large", + corev1.LabelInstanceTypeStable: "t3.large", }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceAWSPodENI: resource.MustParse("1")}, - Limits: v1.ResourceList{v1beta1.ResourceAWSPodENI: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceAWSPodENI: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourceAWSPodENI: resource.MustParse("1")}, }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -333,11 +397,13 @@ var _ = Describe("InstanceTypeProvider", func() { awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{ InstanceTypeOfferings: fake.MakeInstanceOfferings(instances), }) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -376,15 +442,17 @@ var _ = Describe("InstanceTypeProvider", func() { awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{ InstanceTypeOfferings: fake.MakeInstanceOfferings(instances), }) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, Values: []string{ - corev1beta1.CapacityTypeSpot, - corev1beta1.CapacityTypeOnDemand, + karpv1.CapacityTypeSpot, + karpv1.CapacityTypeOnDemand, }, }, }, @@ -392,11 +460,13 @@ var _ = Describe("InstanceTypeProvider", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(generateSpotPricing(cloudProvider, nodePool)) Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -438,12 +508,12 @@ var _ = Describe("InstanceTypeProvider", func() { }) It("should not remove expensive metal instanceTypeOptions if any of the requirement with minValues is provided", func() { // Construct requirements with minValues for capacityType requirement. - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeSpot}, }, MinValues: lo.ToPtr(1), }, @@ -452,9 +522,9 @@ var _ = Describe("InstanceTypeProvider", func() { // Apply requirements and schedule pods. ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, }, }) @@ -476,9 +546,9 @@ var _ = Describe("InstanceTypeProvider", func() { It("should de-prioritize metal", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -495,9 +565,9 @@ var _ = Describe("InstanceTypeProvider", func() { It("should de-prioritize gpu types", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -513,189 +583,262 @@ var _ = Describe("InstanceTypeProvider", func() { }) It("should launch on metal", func() { // add a nodePool requirement for instance type exists to remove our default filter for metal sizes - nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpExists, + nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpExists, }, }) ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{ - v1beta1.LabelInstanceSize: "metal", + v1.LabelInstanceSize: "metal", }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) }) - It("should launch AWS Pod ENI on a compatible instance type", func() { + It("should launch vpc.amazonaws.com/pod-eni on a compatible instance type", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceAWSPodENI: resource.MustParse("1")}, - Limits: v1.ResourceList{v1beta1.ResourceAWSPodENI: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceAWSPodENI: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourceAWSPodENI: resource.MustParse("1")}, }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKey(v1.LabelInstanceTypeStable)) + Expect(node.Labels).To(HaveKey(corev1.LabelInstanceTypeStable)) supportsPodENI := func() bool { - limits, ok := instancetype.Limits[node.Labels[v1.LabelInstanceTypeStable]] + limits, ok := instancetype.Limits[node.Labels[corev1.LabelInstanceTypeStable]] return ok && limits.IsTrunkingCompatible } Expect(supportsPodENI()).To(Equal(true)) }) - It("should launch instances for Nvidia GPU resource requests", func() { + It("should launch vpc.amazonaws.com/PrivateIPv4Address on a compatible instance type", func() { + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "windows2022@latest"}} + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(coretest.PodOptions{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourcePrivateIPv4Address: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourcePrivateIPv4Address: resource.MustParse("1")}, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKey(corev1.LabelInstanceTypeStable)) + limits, ok := instancetype.Limits[node.Labels[corev1.LabelInstanceTypeStable]] + Expect(ok).To(BeTrue()) + Expect(limits.IPv4PerInterface).ToNot(BeZero()) + }) + It("should not launch instance type for vpc.amazonaws.com/PrivateIPv4Address if VPC resource controller doesn't advertise it", func() { + // Create a "test" instance type that has PrivateIPv4Addresses but isn't advertised in the VPC limits config + awsEnv.EC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{ + InstanceTypes: []*ec2.InstanceTypeInfo{ + { + InstanceType: aws.String("test"), + ProcessorInfo: &ec2.ProcessorInfo{ + SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), + }, + VCpuInfo: &ec2.VCpuInfo{ + DefaultCores: aws.Int64(1), + DefaultVCpus: aws.Int64(2), + }, + MemoryInfo: &ec2.MemoryInfo{ + SizeInMiB: aws.Int64(8192), + }, + NetworkInfo: &ec2.NetworkInfo{ + Ipv4AddressesPerInterface: aws.Int64(10), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{{ + NetworkCardIndex: lo.ToPtr(int64(0)), + MaximumNetworkInterfaces: aws.Int64(3), + }}, + }, + SupportedUsageClasses: fake.DefaultSupportedUsageClasses, + }, + }, + }) + awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{ + InstanceTypeOfferings: []*ec2.InstanceTypeOffering{ + { + InstanceType: aws.String("test"), + Location: aws.String("test-zone-1a"), + }, + }, + }) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) + + nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"test"}, + }, + }) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "windows2022@latest"}} + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(coretest.PodOptions{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourcePrivateIPv4Address: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourcePrivateIPv4Address: resource.MustParse("1")}, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should launch instances for nvidia.com/gpu resource requests", func() { nodeNames := sets.NewString() ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pods := []*v1.Pod{ + pods := []*corev1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceNVIDIAGPU: resource.MustParse("1")}, - Limits: v1.ResourceList{v1beta1.ResourceNVIDIAGPU: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceNVIDIAGPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourceNVIDIAGPU: resource.MustParse("1")}, }, }), // Should pack onto same instance coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceNVIDIAGPU: resource.MustParse("2")}, - Limits: v1.ResourceList{v1beta1.ResourceNVIDIAGPU: resource.MustParse("2")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceNVIDIAGPU: resource.MustParse("2")}, + Limits: corev1.ResourceList{v1.ResourceNVIDIAGPU: resource.MustParse("2")}, }, }), // Should pack onto a separate instance coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceNVIDIAGPU: resource.MustParse("4")}, - Limits: v1.ResourceList{v1beta1.ResourceNVIDIAGPU: resource.MustParse("4")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceNVIDIAGPU: resource.MustParse("4")}, + Limits: corev1.ResourceList{v1.ResourceNVIDIAGPU: resource.MustParse("4")}, }, }), } ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) for _, pod := range pods { node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "p3.8xlarge")) + Expect(node.Labels).To(HaveKeyWithValue(corev1.LabelInstanceTypeStable, "p3.8xlarge")) nodeNames.Insert(node.Name) } Expect(nodeNames.Len()).To(Equal(2)) }) - It("should launch instances for Habana GPU resource requests", func() { + It("should launch instances for habana.ai/gaudi resource requests", func() { nodeNames := sets.NewString() ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pods := []*v1.Pod{ + pods := []*corev1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceHabanaGaudi: resource.MustParse("1")}, - Limits: v1.ResourceList{v1beta1.ResourceHabanaGaudi: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceHabanaGaudi: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourceHabanaGaudi: resource.MustParse("1")}, }, }), coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceHabanaGaudi: resource.MustParse("2")}, - Limits: v1.ResourceList{v1beta1.ResourceHabanaGaudi: resource.MustParse("2")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceHabanaGaudi: resource.MustParse("2")}, + Limits: corev1.ResourceList{v1.ResourceHabanaGaudi: resource.MustParse("2")}, }, }), coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceHabanaGaudi: resource.MustParse("4")}, - Limits: v1.ResourceList{v1beta1.ResourceHabanaGaudi: resource.MustParse("4")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceHabanaGaudi: resource.MustParse("4")}, + Limits: corev1.ResourceList{v1.ResourceHabanaGaudi: resource.MustParse("4")}, }, }), } ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) for _, pod := range pods { node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "dl1.24xlarge")) + Expect(node.Labels).To(HaveKeyWithValue(corev1.LabelInstanceTypeStable, "dl1.24xlarge")) nodeNames.Insert(node.Name) } Expect(nodeNames.Len()).To(Equal(1)) }) - It("should launch instances for AWS Neuron resource requests", func() { + It("should launch instances for aws.amazon.com/neuron resource requests", func() { nodeNames := sets.NewString() ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pods := []*v1.Pod{ + pods := []*corev1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("1")}, - Limits: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("1")}, }, }), // Should pack onto same instance coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("2")}, - Limits: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("2")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("2")}, + Limits: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("2")}, }, }), // Should pack onto a separate instance coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("4")}, - Limits: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("4")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("4")}, + Limits: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("4")}, }, }), } ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) for _, pod := range pods { node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "inf1.6xlarge")) + Expect(node.Labels).To(HaveKeyWithValue(corev1.LabelInstanceTypeStable, "inf1.6xlarge")) nodeNames.Insert(node.Name) } Expect(nodeNames.Len()).To(Equal(2)) }) - It("should launch trn1 instances for AWS Neuron resource requests", func() { + It("should launch trn1 instances for aws.amazon.com/neuron resource requests", func() { nodeNames := sets.NewString() - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"trn1.2xlarge"}, }, }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pods := []*v1.Pod{ + pods := []*corev1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("1")}, - Limits: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("1")}, }, }), } ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) for _, pod := range pods { node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "trn1.2xlarge")) + Expect(node.Labels).To(HaveKeyWithValue(corev1.LabelInstanceTypeStable, "trn1.2xlarge")) nodeNames.Insert(node.Name) } Expect(nodeNames.Len()).To(Equal(1)) }) It("should launch instances for vpc.amazonaws.com/efa resource requests", func() { - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"dl1.24xlarge"}, }, }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pods := []*v1.Pod{ + pods := []*corev1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("1")}, - Limits: v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceEFA: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourceEFA: resource.MustParse("1")}, }, }), coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("2")}, - Limits: v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("2")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceEFA: resource.MustParse("2")}, + Limits: corev1.ResourceList{v1.ResourceEFA: resource.MustParse("2")}, }, }), } @@ -703,50 +846,84 @@ var _ = Describe("InstanceTypeProvider", func() { nodes := sets.NewString() for _, pod := range pods { node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "dl1.24xlarge")) + Expect(node.Labels).To(HaveKeyWithValue(corev1.LabelInstanceTypeStable, "dl1.24xlarge")) nodes.Insert(node.Name) } Expect(nodes.Len()).To(Equal(1)) }) + It("should launch instances for amd.com/gpu resource requests", func() { + nodeNames := sets.NewString() + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pods := []*corev1.Pod{ + coretest.UnschedulablePod(coretest.PodOptions{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceAMDGPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourceAMDGPU: resource.MustParse("1")}, + }, + }), + // Should pack onto same instance + coretest.UnschedulablePod(coretest.PodOptions{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceAMDGPU: resource.MustParse("2")}, + Limits: corev1.ResourceList{v1.ResourceAMDGPU: resource.MustParse("2")}, + }, + }), + // Should pack onto a separate instance + coretest.UnschedulablePod(coretest.PodOptions{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceAMDGPU: resource.MustParse("4")}, + Limits: corev1.ResourceList{v1.ResourceAMDGPU: resource.MustParse("4")}, + }, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(corev1.LabelInstanceTypeStable, "g4ad.16xlarge")) + nodeNames.Insert(node.Name) + } + Expect(nodeNames.Len()).To(Equal(2)) + }) It("should not launch instances w/ instance storage for ephemeral storage resource requests when exceeding blockDeviceMapping", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceEphemeralStorage: resource.MustParse("5000Gi")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("5000Gi")}, }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) }) It("should launch instances w/ instance storage for ephemeral storage resource requests when disks are mounted for ephemeral-storage", func() { - nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1beta1.InstanceStorePolicyRAID0) + nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1.InstanceStorePolicyRAID0) ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceEphemeralStorage: resource.MustParse("5000Gi")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("5000Gi")}, }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("m6idn.32xlarge")) + Expect(node.Labels[corev1.LabelInstanceTypeStable]).To(Equal("m6idn.32xlarge")) Expect(*node.Status.Capacity.StorageEphemeral()).To(Equal(resource.MustParse("7600G"))) }) It("should not set pods to 110 if using ENI-based pod density", func() { - instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) - for _, info := range instanceInfo { - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} + for _, info := range instanceInfo.InstanceTypes { + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -754,22 +931,22 @@ var _ = Describe("InstanceTypeProvider", func() { } }) It("should set pods to 110 if AMI Family doesn't support", func() { - instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) - - for _, info := range instanceInfo { - amiFamily := amifamily.GetAMIFamily(windowsNodeClass.Spec.AMIFamily, &amifamily.Options{}) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} + for _, info := range instanceInfo.InstanceTypes { + amiFamily := amifamily.GetAMIFamily(windowsNodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, windowsNodeClass.Spec.BlockDeviceMappings, windowsNodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -778,7 +955,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) Context("Metrics", func() { It("should expose vcpu metrics for instance types", func() { - instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodeClass.Spec.Kubelet, nodeClass) Expect(err).To(BeNil()) Expect(len(instanceTypes)).To(BeNumerically(">", 0)) for _, it := range instanceTypes { @@ -792,7 +969,7 @@ var _ = Describe("InstanceTypeProvider", func() { } }) It("should expose memory metrics for instance types", func() { - instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodeClass.Spec.Kubelet, nodeClass) Expect(err).To(BeNil()) Expect(len(instanceTypes)).To(BeNumerically(">", 0)) for _, it := range instanceTypes { @@ -806,15 +983,15 @@ var _ = Describe("InstanceTypeProvider", func() { } }) It("should expose availability metrics for instance types", func() { - instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodeClass.Spec.Kubelet, nodeClass) Expect(err).To(BeNil()) Expect(len(instanceTypes)).To(BeNumerically(">", 0)) for _, it := range instanceTypes { for _, of := range it.Offerings { metric, ok := FindMetricWithLabelValues("karpenter_cloudprovider_instance_type_offering_available", map[string]string{ "instance_type": it.Name, - "capacity_type": of.CapacityType, - "zone": of.Zone, + "capacity_type": of.Requirements.Get(karpv1.CapacityTypeLabelKey).Any(), + "zone": of.Requirements.Get(corev1.LabelTopologyZone).Any(), }) Expect(ok).To(BeTrue()) Expect(metric).To(Not(BeNil())) @@ -824,15 +1001,15 @@ var _ = Describe("InstanceTypeProvider", func() { } }) It("should expose pricing metrics for instance types", func() { - instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodeClass.Spec.Kubelet, nodeClass) Expect(err).To(BeNil()) Expect(len(instanceTypes)).To(BeNumerically(">", 0)) for _, it := range instanceTypes { for _, of := range it.Offerings { metric, ok := FindMetricWithLabelValues("karpenter_cloudprovider_instance_type_offering_price_estimate", map[string]string{ "instance_type": it.Name, - "capacity_type": of.CapacityType, - "zone": of.Zone, + "capacity_type": of.Requirements.Get(karpv1.CapacityTypeLabelKey).Any(), + "zone": of.Requirements.Get(corev1.LabelTopologyZone).Any(), }) Expect(ok).To(BeTrue()) Expect(metric).To(Not(BeNil())) @@ -843,19 +1020,23 @@ var _ = Describe("InstanceTypeProvider", func() { }) }) It("should launch instances in local zones", func() { + nodeClass.Status.Subnets = []v1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a-local", + }, + } ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ - NodeRequirements: []v1.NodeSelectorRequirement{{ - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, + NodeRequirements: []corev1.NodeSelectorRequirement{{ + Key: corev1.LabelTopologyZone, + Operator: corev1.NodeSelectorOpIn, Values: []string{"test-zone-1a-local"}, }}, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) - }) - Context("Overhead", func() { var info *ec2.InstanceTypeInfo BeforeEach(func() { @@ -864,27 +1045,28 @@ var _ = Describe("InstanceTypeProvider", func() { })) var ok bool - instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) - info, ok = lo.Find(instanceInfo, func(i *ec2.InstanceTypeInfo) bool { + info, ok = lo.Find(instanceInfo.InstanceTypes, func(i *ec2.InstanceTypeInfo) bool { return aws.StringValue(i.InstanceType) == "m5.xlarge" }) Expect(ok).To(BeTrue()) }) Context("System Reserved Resources", func() { It("should use defaults when no kubelet is specified", func() { - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -893,25 +1075,25 @@ var _ = Describe("InstanceTypeProvider", func() { Expect(it.Overhead.SystemReserved.StorageEphemeral().String()).To(Equal("0")) }) It("should override system reserved cpus when specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceCPU): "2", - string(v1.ResourceMemory): "20Gi", - string(v1.ResourceEphemeralStorage): "10Gi", + string(corev1.ResourceCPU): "2", + string(corev1.ResourceMemory): "20Gi", + string(corev1.ResourceEphemeralStorage): "10Gi", }, } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -922,19 +1104,19 @@ var _ = Describe("InstanceTypeProvider", func() { }) Context("Kube Reserved Resources", func() { It("should use defaults when no kubelet is specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{} - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -943,30 +1125,30 @@ var _ = Describe("InstanceTypeProvider", func() { Expect(it.Overhead.KubeReserved.StorageEphemeral().String()).To(Equal("1Gi")) }) It("should override kube reserved when specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceCPU): "1", - string(v1.ResourceMemory): "20Gi", - string(v1.ResourceEphemeralStorage): "1Gi", + string(corev1.ResourceCPU): "1", + string(corev1.ResourceMemory): "20Gi", + string(corev1.ResourceEphemeralStorage): "1Gi", }, KubeReserved: map[string]string{ - string(v1.ResourceCPU): "2", - string(v1.ResourceMemory): "10Gi", - string(v1.ResourceEphemeralStorage): "2Gi", + string(corev1.ResourceCPU): "2", + string(corev1.ResourceMemory): "10Gi", + string(corev1.ResourceEphemeralStorage): "2Gi", }, } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -983,116 +1165,116 @@ var _ = Describe("InstanceTypeProvider", func() { }) Context("Eviction Hard", func() { It("should override eviction threshold when specified as a quantity", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceMemory): "20Gi", + string(corev1.ResourceMemory): "20Gi", }, KubeReserved: map[string]string{ - string(v1.ResourceMemory): "10Gi", + string(corev1.ResourceMemory): "10Gi", }, EvictionHard: map[string]string{ instancetype.MemoryAvailable: "500Mi", }, } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold when specified as a percentage value", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceMemory): "20Gi", + string(corev1.ResourceMemory): "20Gi", }, KubeReserved: map[string]string{ - string(v1.ResourceMemory): "10Gi", + string(corev1.ResourceMemory): "10Gi", }, EvictionHard: map[string]string{ instancetype.MemoryAvailable: "10%", }, } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold disabled when specified as 100%", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceMemory): "20Gi", + string(corev1.ResourceMemory): "20Gi", }, KubeReserved: map[string]string{ - string(v1.ResourceMemory): "10Gi", + string(corev1.ResourceMemory): "10Gi", }, EvictionHard: map[string]string{ instancetype.MemoryAvailable: "100%", }, } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should used default eviction threshold for memory when evictionHard not specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceMemory): "20Gi", + string(corev1.ResourceMemory): "20Gi", }, KubeReserved: map[string]string{ - string(v1.ResourceMemory): "10Gi", + string(corev1.ResourceMemory): "10Gi", }, EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "50Mi", }, } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -1101,41 +1283,41 @@ var _ = Describe("InstanceTypeProvider", func() { }) Context("Eviction Soft", func() { It("should override eviction threshold when specified as a quantity", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceMemory): "20Gi", + string(corev1.ResourceMemory): "20Gi", }, KubeReserved: map[string]string{ - string(v1.ResourceMemory): "10Gi", + string(corev1.ResourceMemory): "10Gi", }, EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "500Mi", }, } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold when specified as a percentage value", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceMemory): "20Gi", + string(corev1.ResourceMemory): "20Gi", }, KubeReserved: map[string]string{ - string(v1.ResourceMemory): "10Gi", + string(corev1.ResourceMemory): "10Gi", }, EvictionHard: map[string]string{ instancetype.MemoryAvailable: "5%", @@ -1144,60 +1326,60 @@ var _ = Describe("InstanceTypeProvider", func() { instancetype.MemoryAvailable: "10%", }, } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold disabled when specified as 100%", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceMemory): "20Gi", + string(corev1.ResourceMemory): "20Gi", }, KubeReserved: map[string]string{ - string(v1.ResourceMemory): "10Gi", + string(corev1.ResourceMemory): "10Gi", }, EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "100%", }, } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should ignore eviction threshold when using Bottlerocket AMI", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceMemory): "20Gi", + string(corev1.ResourceMemory): "20Gi", }, KubeReserved: map[string]string{ - string(v1.ResourceMemory): "10Gi", + string(corev1.ResourceMemory): "10Gi", }, EvictionHard: map[string]string{ instancetype.MemoryAvailable: "1Gi", @@ -1206,18 +1388,18 @@ var _ = Describe("InstanceTypeProvider", func() { instancetype.MemoryAvailable: "10Gi", }, } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -1225,19 +1407,19 @@ var _ = Describe("InstanceTypeProvider", func() { }) }) It("should take the default eviction threshold when none is specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{} - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -1246,12 +1428,12 @@ var _ = Describe("InstanceTypeProvider", func() { Expect(it.Overhead.EvictionThreshold.StorageEphemeral().AsApproximateFloat64()).To(BeNumerically("~", resources.Quantity("2Gi").AsApproximateFloat64())) }) It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceMemory): "20Gi", + string(corev1.ResourceMemory): "20Gi", }, KubeReserved: map[string]string{ - string(v1.ResourceMemory): "10Gi", + string(corev1.ResourceMemory): "10Gi", }, EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "3Gi", @@ -1260,30 +1442,30 @@ var _ = Describe("InstanceTypeProvider", func() { instancetype.MemoryAvailable: "1Gi", }, } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("3Gi")) }) It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceMemory): "20Gi", + string(corev1.ResourceMemory): "20Gi", }, KubeReserved: map[string]string{ - string(v1.ResourceMemory): "10Gi", + string(corev1.ResourceMemory): "10Gi", }, EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "2%", @@ -1292,30 +1474,30 @@ var _ = Describe("InstanceTypeProvider", func() { instancetype.MemoryAvailable: "5%", }, } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.05, 10)) }) It("should take the greater of evictionHard and evictionSoft for overhead with mixed percentage/value", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceMemory): "20Gi", + string(corev1.ResourceMemory): "20Gi", }, KubeReserved: map[string]string{ - string(v1.ResourceMemory): "10Gi", + string(corev1.ResourceMemory): "10Gi", }, EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "10%", @@ -1324,18 +1506,18 @@ var _ = Describe("InstanceTypeProvider", func() { instancetype.MemoryAvailable: "1Gi", }, } - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -1343,66 +1525,67 @@ var _ = Describe("InstanceTypeProvider", func() { }) }) It("should default max pods based off of network interfaces", func() { - instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) - for _, info := range instanceInfo { + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} + for _, info := range instanceInfo.InstanceTypes { if *info.InstanceType == "t3.large" { - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 35)) } if *info.InstanceType == "m6idn.32xlarge" { - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) - Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 345)) + Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 394)) } } }) It("should set max-pods to user-defined value if specified", func() { - instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - MaxPods: ptr.Int32(10), + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + MaxPods: lo.ToPtr(int32(10)), } - for _, info := range instanceInfo { - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + for _, info := range instanceInfo.InstanceTypes { + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -1410,24 +1593,24 @@ var _ = Describe("InstanceTypeProvider", func() { } }) It("should override max-pods value", func() { - instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - MaxPods: ptr.Int32(10), + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + MaxPods: lo.ToPtr(int32(10)), } - for _, info := range instanceInfo { - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + for _, info := range instanceInfo.InstanceTypes { + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -1439,24 +1622,25 @@ var _ = Describe("InstanceTypeProvider", func() { ReservedENIs: lo.ToPtr(1), })) - instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) - t3Large, ok := lo.Find(instanceInfo, func(info *ec2.InstanceTypeInfo) bool { + t3Large, ok := lo.Find(instanceInfo.InstanceTypes, func(info *ec2.InstanceTypeInfo) bool { return *info.InstanceType == "t3.large" }) Expect(ok).To(Equal(true)) - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} it := instancetype.NewInstanceType(ctx, t3Large, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -1473,24 +1657,25 @@ var _ = Describe("InstanceTypeProvider", func() { ReservedENIs: lo.ToPtr(1_000_000), })) - instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) - t3Large, ok := lo.Find(instanceInfo, func(info *ec2.InstanceTypeInfo) bool { + t3Large, ok := lo.Find(instanceInfo.InstanceTypes, func(info *ec2.InstanceTypeInfo) bool { return *info.InstanceType == "t3.large" }) Expect(ok).To(Equal(true)) - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} it := instancetype.NewInstanceType(ctx, t3Large, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -1504,76 +1689,76 @@ var _ = Describe("InstanceTypeProvider", func() { Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", maxPods)) }) It("should override pods-per-core value", func() { - instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - PodsPerCore: ptr.Int32(1), + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + PodsPerCore: lo.ToPtr(int32(1)), } - for _, info := range instanceInfo { - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + for _, info := range instanceInfo.InstanceTypes { + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) - Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", ptr.Int64Value(info.VCpuInfo.DefaultVCpus))) + Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", lo.FromPtr(info.VCpuInfo.DefaultVCpus))) } }) It("should take the minimum of pods-per-core and max-pods", func() { - instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - PodsPerCore: ptr.Int32(4), - MaxPods: ptr.Int32(20), + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + PodsPerCore: lo.ToPtr(int32(4)), + MaxPods: lo.ToPtr(int32(20)), } - for _, info := range instanceInfo { - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + for _, info := range instanceInfo.InstanceTypes { + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) - Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", lo.Min([]int64{20, ptr.Int64Value(info.VCpuInfo.DefaultVCpus) * 4}))) + Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", lo.Min([]int64{20, lo.FromPtr(info.VCpuInfo.DefaultVCpus) * 4}))) } }) It("should ignore pods-per-core when using Bottlerocket AMI", func() { - instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - PodsPerCore: ptr.Int32(1), + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + PodsPerCore: lo.ToPtr(int32(1)), } - for _, info := range instanceInfo { - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + for _, info := range instanceInfo.InstanceTypes { + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -1582,47 +1767,47 @@ var _ = Describe("InstanceTypeProvider", func() { } }) It("should take limited pod density to be the default pods number when pods-per-core is 0", func() { - instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - PodsPerCore: ptr.Int32(0), + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + PodsPerCore: lo.ToPtr(int32(0)), } - for _, info := range instanceInfo { + for _, info := range instanceInfo.InstanceTypes { if *info.InstanceType == "t3.large" { - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 35)) } if *info.InstanceType == "m6idn.32xlarge" { - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) - Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 345)) + Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 394)) } } }) @@ -1716,21 +1901,21 @@ var _ = Describe("InstanceTypeProvider", func() { }) Context("Insufficient Capacity Error Cache", func() { It("should launch instances of different type on second reconciliation attempt with Insufficient Capacity Error Cache fallback", func() { - awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "inf1.6xlarge", Zone: "test-zone-1a"}}) + awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: karpv1.CapacityTypeOnDemand, InstanceType: "inf1.6xlarge", Zone: "test-zone-1a"}}) ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pods := []*v1.Pod{ + pods := []*corev1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("1")}, - Limits: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("1")}, + NodeSelector: map[string]string{corev1.LabelTopologyZone: "test-zone-1a"}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("1")}, }, }), coretest.UnschedulablePod(coretest.PodOptions{ - NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("1")}, - Limits: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("1")}, + NodeSelector: map[string]string{corev1.LabelTopologyZone: "test-zone-1a"}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("1")}, }, }), } @@ -1743,24 +1928,24 @@ var _ = Describe("InstanceTypeProvider", func() { ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) for _, pod := range pods { node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1beta1.LabelInstanceAcceleratorName, "inferentia")) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceAcceleratorName, "inferentia")) nodeNames.Insert(node.Name) } Expect(nodeNames.Len()).To(Equal(2)) }) It("should launch instances in a different zone on second reconciliation attempt with Insufficient Capacity Error Cache fallback", func() { - awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "p3.8xlarge", Zone: "test-zone-1a"}}) + awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: karpv1.CapacityTypeOnDemand, InstanceType: "p3.8xlarge", Zone: "test-zone-1a"}}) pod := coretest.UnschedulablePod(coretest.PodOptions{ - NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "p3.8xlarge"}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceNVIDIAGPU: resource.MustParse("1")}, - Limits: v1.ResourceList{v1beta1.ResourceNVIDIAGPU: resource.MustParse("1")}, + NodeSelector: map[string]string{corev1.LabelInstanceTypeStable: "p3.8xlarge"}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceNVIDIAGPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourceNVIDIAGPU: resource.MustParse("1")}, }, }) - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + pod.Spec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1a"}}, + Weight: 1, Preference: corev1.NodeSelectorTerm{MatchExpressions: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelTopologyZone, Operator: corev1.NodeSelectorOpIn, Values: []string{"test-zone-1a"}}, }}, }, }}} @@ -1772,28 +1957,28 @@ var _ = Describe("InstanceTypeProvider", func() { ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(SatisfyAll( - HaveKeyWithValue(v1.LabelInstanceTypeStable, "p3.8xlarge"), - HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1b"))) + HaveKeyWithValue(corev1.LabelInstanceTypeStable, "p3.8xlarge"), + HaveKeyWithValue(corev1.LabelTopologyZone, "test-zone-1b"))) }) It("should launch smaller instances than optimal if larger instance launch results in Insufficient Capacity Error", func() { awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{ - {CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, + {CapacityType: karpv1.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, }) - nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceType, - Operator: v1.NodeSelectorOpIn, + nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceType, + Operator: corev1.NodeSelectorOpIn, Values: []string{"m5.large", "m5.xlarge"}, }, }) - pods := []*v1.Pod{} + pods := []*corev1.Pod{} for i := 0; i < 2; i++ { pods = append(pods, coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, }, NodeSelector: map[string]string{ - v1.LabelTopologyZone: "test-zone-1a", + corev1.LabelTopologyZone: "test-zone-1a", }, })) } @@ -1806,41 +1991,41 @@ var _ = Describe("InstanceTypeProvider", func() { ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) for _, pod := range pods { node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("m5.large")) + Expect(node.Labels[corev1.LabelInstanceTypeStable]).To(Equal("m5.large")) } }) It("should launch instances on later reconciliation attempt with Insufficient Capacity Error Cache expiry", func() { - awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "inf1.6xlarge", Zone: "test-zone-1a"}}) + awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: karpv1.CapacityTypeOnDemand, InstanceType: "inf1.6xlarge", Zone: "test-zone-1a"}}) ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ - NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "inf1.6xlarge"}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("2")}, - Limits: v1.ResourceList{v1beta1.ResourceAWSNeuron: resource.MustParse("2")}, + NodeSelector: map[string]string{corev1.LabelInstanceTypeStable: "inf1.6xlarge"}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("2")}, + Limits: corev1.ResourceList{v1.ResourceAWSNeuron: resource.MustParse("2")}, }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) // capacity shortage is over - expire the item from the cache and try again awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{}) - awsEnv.UnavailableOfferingsCache.Delete("inf1.6xlarge", "test-zone-1a", corev1beta1.CapacityTypeOnDemand) + awsEnv.UnavailableOfferingsCache.Delete("inf1.6xlarge", "test-zone-1a", karpv1.CapacityTypeOnDemand) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "inf1.6xlarge")) + Expect(node.Labels).To(HaveKeyWithValue(corev1.LabelInstanceTypeStable, "inf1.6xlarge")) }) It("should launch instances in a different zone on second reconciliation attempt with Insufficient Capacity Error Cache fallback (Habana)", func() { - awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "dl1.24xlarge", Zone: "test-zone-1a"}}) + awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: karpv1.CapacityTypeOnDemand, InstanceType: "dl1.24xlarge", Zone: "test-zone-1a"}}) pod := coretest.UnschedulablePod(coretest.PodOptions{ - NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "dl1.24xlarge"}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceHabanaGaudi: resource.MustParse("1")}, - Limits: v1.ResourceList{v1beta1.ResourceHabanaGaudi: resource.MustParse("1")}, + NodeSelector: map[string]string{corev1.LabelInstanceTypeStable: "dl1.24xlarge"}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceHabanaGaudi: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourceHabanaGaudi: resource.MustParse("1")}, }, }) - pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + pod.Spec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ { - Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1a"}}, + Weight: 1, Preference: corev1.NodeSelectorTerm{MatchExpressions: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelTopologyZone, Operator: corev1.NodeSelectorOpIn, Values: []string{"test-zone-1a"}}, }}, }, }}} @@ -1852,19 +2037,19 @@ var _ = Describe("InstanceTypeProvider", func() { ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(SatisfyAll( - HaveKeyWithValue(v1.LabelInstanceTypeStable, "dl1.24xlarge"), - HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1b"))) + HaveKeyWithValue(corev1.LabelInstanceTypeStable, "dl1.24xlarge"), + HaveKeyWithValue(corev1.LabelTopologyZone, "test-zone-1b"))) }) It("should launch on-demand capacity if flexible to both spot and on-demand, but spot is unavailable", func() { Expect(awsEnv.EC2API.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}, func(dito *ec2.DescribeInstanceTypesOutput, b bool) bool { for _, it := range dito.InstanceTypes { - awsEnv.EC2API.InsufficientCapacityPools.Add(fake.CapacityPool{CapacityType: corev1beta1.CapacityTypeSpot, InstanceType: aws.StringValue(it.InstanceType), Zone: "test-zone-1a"}) + awsEnv.EC2API.InsufficientCapacityPools.Add(fake.CapacityPool{CapacityType: karpv1.CapacityTypeSpot, InstanceType: aws.StringValue(it.InstanceType), Zone: "test-zone-1a"}) } return true })).To(Succeed()) - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot, corev1beta1.CapacityTypeOnDemand}}}, - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1a"}}}, + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: karpv1.CapacityTypeLabelKey, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.CapacityTypeSpot, karpv1.CapacityTypeOnDemand}}}, + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: corev1.LabelTopologyZone, Operator: corev1.NodeSelectorOpIn, Values: []string{"test-zone-1a"}}}, } // Spot Unavailable ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1875,49 +2060,48 @@ var _ = Describe("InstanceTypeProvider", func() { ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) // Fallback to OD node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(corev1beta1.CapacityTypeLabelKey, corev1beta1.CapacityTypeOnDemand)) + Expect(node.Labels).To(HaveKeyWithValue(karpv1.CapacityTypeLabelKey, karpv1.CapacityTypeOnDemand)) }) It("should return all instance types, even though with no offerings due to Insufficient Capacity Error", func() { awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{ - {CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, - {CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, - {CapacityType: corev1beta1.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, - {CapacityType: corev1beta1.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, + {CapacityType: karpv1.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, + {CapacityType: karpv1.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, + {CapacityType: karpv1.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, + {CapacityType: karpv1.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, }) nodePool.Spec.Template.Spec.Requirements = nil - nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceType, - Operator: v1.NodeSelectorOpIn, + nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceType, + Operator: corev1.NodeSelectorOpIn, Values: []string{"m5.xlarge"}, }, }, ) - nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, + nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, Values: []string{"spot", "on-demand"}, }, }) ExpectApplied(ctx, env.Client, nodePool, nodeClass) - for _, ct := range []string{corev1beta1.CapacityTypeOnDemand, corev1beta1.CapacityTypeSpot} { + for _, ct := range []string{karpv1.CapacityTypeOnDemand, karpv1.CapacityTypeSpot} { for _, zone := range []string{"test-zone-1a", "test-zone-1b"} { ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, }, NodeSelector: map[string]string{ - corev1beta1.CapacityTypeLabelKey: ct, - v1.LabelTopologyZone: zone, + karpv1.CapacityTypeLabelKey: ct, + corev1.LabelTopologyZone: zone, }, })) } } - awsEnv.InstanceTypeCache.Flush() instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, nodePool) Expect(err).To(BeNil()) instanceTypeNames := sets.NewString() @@ -1937,16 +2121,16 @@ var _ = Describe("InstanceTypeProvider", func() { pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(corev1beta1.CapacityTypeLabelKey, corev1beta1.CapacityTypeOnDemand)) + Expect(node.Labels).To(HaveKeyWithValue(karpv1.CapacityTypeLabelKey, karpv1.CapacityTypeOnDemand)) }) It("should launch spot capacity if flexible to both spot and on demand", func() { - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot, corev1beta1.CapacityTypeOnDemand}}}} + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: karpv1.CapacityTypeLabelKey, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.CapacityTypeSpot, karpv1.CapacityTypeOnDemand}}}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(corev1beta1.CapacityTypeLabelKey, corev1beta1.CapacityTypeSpot)) + Expect(node.Labels).To(HaveKeyWithValue(karpv1.CapacityTypeLabelKey, karpv1.CapacityTypeSpot)) }) It("should fail to launch capacity when there is no zonal availability for spot", func() { now := time.Now() @@ -1962,10 +2146,10 @@ var _ = Describe("InstanceTypeProvider", func() { }) Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot}}}, - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"m5.large"}}}, - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1b"}}}, + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: karpv1.CapacityTypeLabelKey, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.CapacityTypeSpot}}}, + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: corev1.LabelInstanceTypeStable, Operator: corev1.NodeSelectorOpIn, Values: []string{"m5.large"}}}, + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: corev1.LabelTopologyZone, Operator: corev1.NodeSelectorOpIn, Values: []string{"test-zone-1b"}}}, } // Instance type with no zonal availability for spot shouldn't be scheduled @@ -1989,33 +2173,33 @@ var _ = Describe("InstanceTypeProvider", func() { Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) // not restricting to the zone so we can get any zone - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot}}}, - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"m5.large"}}}, + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: karpv1.CapacityTypeLabelKey, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.CapacityTypeSpot}}}, + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: corev1.LabelInstanceTypeStable, Operator: corev1.NodeSelectorOpIn, Values: []string{"m5.large"}}}, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(corev1beta1.NodePoolLabelKey, nodePool.Name)) + Expect(node.Labels).To(HaveKeyWithValue(karpv1.NodePoolLabelKey, nodePool.Name)) }) }) Context("Ephemeral Storage", func() { BeforeEach(func() { - nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyAL2) - nodeClass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} + nodeClass.Spec.BlockDeviceMappings = []*v1.BlockDeviceMapping{ { DeviceName: aws.String("/dev/xvda"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ SnapshotID: aws.String("snap-xxxxxxxx"), }, }, } }) It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for custom AMIs", func() { - nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyCustom) - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ { Tags: map[string]string{ "*": "*", @@ -2048,7 +2232,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) }) It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for AL2023 Root volume", func() { - nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2023@latest"}} awsEnv.LaunchTemplateProvider.CABundle = lo.ToPtr("Y2EtYnVuZGxlCg==") awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(lo.ToPtr("10.100.0.0/16")) ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -2064,7 +2248,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) }) It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for Bottlerocket Root volume", func() { - nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyBottlerocket) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} nodeClass.Spec.BlockDeviceMappings[0].DeviceName = aws.String("/dev/xvdb") ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() @@ -2079,21 +2263,6 @@ var _ = Describe("InstanceTypeProvider", func() { Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.SnapshotId).To(Equal("snap-xxxxxxxx")) }) }) - It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for Ubuntu Root volume", func() { - nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyUbuntu) - nodeClass.Spec.BlockDeviceMappings[0].DeviceName = aws.String("/dev/sda1") - ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - Expect(*node.Status.Capacity.StorageEphemeral()).To(Equal(resource.MustParse("20Gi"))) - Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(BeNumerically(">=", 1)) - awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.ForEach(func(ltInput *ec2.CreateLaunchTemplateInput) { - Expect(ltInput.LaunchTemplateData.BlockDeviceMappings).To(HaveLen(1)) - Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].DeviceName).To(Equal("/dev/sda1")) - Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.SnapshotId).To(Equal("snap-xxxxxxxx")) - }) - }) }) Context("Metadata Options", func() { It("should default metadata options on generated launch template", func() { @@ -2105,12 +2274,12 @@ var _ = Describe("InstanceTypeProvider", func() { awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.ForEach(func(ltInput *ec2.CreateLaunchTemplateInput) { Expect(*ltInput.LaunchTemplateData.MetadataOptions.HttpEndpoint).To(Equal(ec2.LaunchTemplateInstanceMetadataEndpointStateEnabled)) Expect(*ltInput.LaunchTemplateData.MetadataOptions.HttpProtocolIpv6).To(Equal(ec2.LaunchTemplateInstanceMetadataProtocolIpv6Disabled)) - Expect(*ltInput.LaunchTemplateData.MetadataOptions.HttpPutResponseHopLimit).To(Equal(int64(2))) + Expect(*ltInput.LaunchTemplateData.MetadataOptions.HttpPutResponseHopLimit).To(Equal(int64(1))) Expect(*ltInput.LaunchTemplateData.MetadataOptions.HttpTokens).To(Equal(ec2.LaunchTemplateHttpTokensStateRequired)) }) }) It("should set metadata options on generated launch template from nodePool configuration", func() { - nodeClass.Spec.MetadataOptions = &v1beta1.MetadataOptions{ + nodeClass.Spec.MetadataOptions = &v1.MetadataOptions{ HTTPEndpoint: aws.String(ec2.LaunchTemplateInstanceMetadataEndpointStateDisabled), HTTPProtocolIPv6: aws.String(ec2.LaunchTemplateInstanceMetadataProtocolIpv6Enabled), HTTPPutResponseHopLimit: aws.Int64(1), @@ -2138,17 +2307,20 @@ var _ = Describe("InstanceTypeProvider", func() { // kubelet.evictionHard // kubelet.evictionSoft // kubelet.maxPods - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - KubeReserved: map[string]string{string(v1.ResourceCPU): "1"}, - SystemReserved: map[string]string{string(v1.ResourceCPU): "1"}, + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + KubeReserved: map[string]string{string(corev1.ResourceCPU): "1"}, + SystemReserved: map[string]string{string(corev1.ResourceCPU): "1"}, EvictionHard: map[string]string{"memory.available": "5%"}, EvictionSoft: map[string]string{"nodefs.available": "10%"}, - MaxPods: aws.Int32(10), + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "nodefs.available": {Duration: time.Minute}, + }, + MaxPods: aws.Int32(10), } - kubeletChanges := []*corev1beta1.KubeletConfiguration{ + kubeletChanges := []*v1.KubeletConfiguration{ {}, // Testing the base case black EC2NodeClass - {KubeReserved: map[string]string{string(v1.ResourceCPU): "20"}}, - {SystemReserved: map[string]string{string(v1.ResourceMemory): "10Gi"}}, + {KubeReserved: map[string]string{string(corev1.ResourceCPU): "20"}}, + {SystemReserved: map[string]string{string(corev1.ResourceMemory): "10Gi"}}, {EvictionHard: map[string]string{"memory.available": "52%"}}, {EvictionSoft: map[string]string{"nodefs.available": "132%"}}, {MaxPods: aws.Int32(20)}, @@ -2165,12 +2337,12 @@ var _ = Describe("InstanceTypeProvider", func() { sorted := nodePool.DeepCopy() for _, change := range kubeletChanges { nodePool = sorted.DeepCopy() - Expect(mergo.Merge(nodePool.Spec.Template.Spec.Kubelet, change, mergo.WithOverride, mergo.WithSliceDeepCopy)).To(BeNil()) + Expect(mergo.Merge(nodeClass.Spec.Kubelet, change, mergo.WithOverride, mergo.WithSliceDeepCopy)).To(BeNil()) // Calling the provider and storing the instance type list to the instancetype provider cache - _, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + _, err := awsEnv.InstanceTypesProvider.List(ctx, nodeClass.Spec.Kubelet, nodeClass) Expect(err).To(BeNil()) // We are making sure to pull from the cache - instancetypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + instancetypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodeClass.Spec.Kubelet, nodeClass) Expect(err).To(BeNil()) sort.Slice(instancetypes, func(x int, y int) bool { return instancetypes[x].Name < instancetypes[y].Name @@ -2184,44 +2356,44 @@ var _ = Describe("InstanceTypeProvider", func() { It("changes to nodeclass fields should result in a different set of instances types", func() { // We should expect these nodeclass fields to change the result of the instance type // nodeClass.instanceStorePolicy - // nodeClass.amiFamily + // nodeClass.amiSelectorTerms (alias) // nodeClass.blockDeviceMapping.rootVolume // nodeClass.blockDeviceMapping.volumeSize // nodeClass.blockDeviceMapping.deviceName - nodeClass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ + nodeClass.Spec.BlockDeviceMappings = []*v1.BlockDeviceMapping{ { DeviceName: lo.ToPtr("/dev/xvda"), - EBS: &v1beta1.BlockDevice{VolumeSize: resource.NewScaledQuantity(10, resource.Giga)}, + EBS: &v1.BlockDevice{VolumeSize: resource.NewScaledQuantity(10, resource.Giga)}, RootVolume: false, }, } - nodeClassChanges := []*v1beta1.EC2NodeClass{ + nodeClassChanges := []*v1.EC2NodeClass{ {}, // Testing the base case black EC2NodeClass - {Spec: v1beta1.EC2NodeClassSpec{InstanceStorePolicy: lo.ToPtr(v1beta1.InstanceStorePolicyRAID0)}}, - {Spec: v1beta1.EC2NodeClassSpec{AMIFamily: &v1beta1.AMIFamilyUbuntu}}, + {Spec: v1.EC2NodeClassSpec{InstanceStorePolicy: lo.ToPtr(v1.InstanceStorePolicyRAID0)}}, + {Spec: v1.EC2NodeClassSpec{AMISelectorTerms: []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}}}}, { - Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{ { DeviceName: lo.ToPtr("/dev/sda1"), - EBS: &v1beta1.BlockDevice{VolumeSize: resource.NewScaledQuantity(10, resource.Giga)}, + EBS: &v1.BlockDevice{VolumeSize: resource.NewScaledQuantity(10, resource.Giga)}, RootVolume: true, }, }, }}, { - Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{ { DeviceName: lo.ToPtr("/dev/xvda"), - EBS: &v1beta1.BlockDevice{VolumeSize: resource.NewScaledQuantity(10, resource.Giga)}, + EBS: &v1.BlockDevice{VolumeSize: resource.NewScaledQuantity(10, resource.Giga)}, RootVolume: true, }, }, }}, { - Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{ { DeviceName: lo.ToPtr("/dev/xvda"), - EBS: &v1beta1.BlockDevice{VolumeSize: resource.NewScaledQuantity(20, resource.Giga)}, + EBS: &v1.BlockDevice{VolumeSize: resource.NewScaledQuantity(20, resource.Giga)}, RootVolume: false, }, }, @@ -2242,10 +2414,10 @@ var _ = Describe("InstanceTypeProvider", func() { nodeClass = sorted.DeepCopy() Expect(mergo.Merge(nodeClass, change, mergo.WithOverride)).To(BeNil()) // Calling the provider and storing the instance type list to the instancetype provider cache - _, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + _, err := awsEnv.InstanceTypesProvider.List(ctx, nodeClass.Spec.Kubelet, nodeClass) Expect(err).To(BeNil()) // We are making sure to pull from the cache - instanetypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + instanetypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodeClass.Spec.Kubelet, nodeClass) Expect(err).To(BeNil()) sort.Slice(instanetypes, func(x int, y int) bool { return instanetypes[x].Name < instanetypes[y].Name @@ -2257,6 +2429,41 @@ var _ = Describe("InstanceTypeProvider", func() { uniqueInstanceTypeList(instanceTypeResult) }) }) + It("should not cause data races when calling List() simultaneously", func() { + mu := sync.RWMutex{} + var instanceTypeOrder []string + wg := sync.WaitGroup{} + for i := 0; i < 10000; i++ { + wg.Add(1) + go func() { + defer wg.Done() + defer GinkgoRecover() + instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, &v1.KubeletConfiguration{}, nodeClass) + Expect(err).ToNot(HaveOccurred()) + + // Sort everything in parallel and ensure that we don't get data races + sort.Slice(instanceTypes, func(i, j int) bool { + return instanceTypes[i].Name < instanceTypes[j].Name + }) + // Get the ordering of the instance types based on name + tempInstanceTypeOrder := lo.Map(instanceTypes, func(i *corecloudprovider.InstanceType, _ int) string { + return i.Name + }) + // Expect that all the elements in the instance type list are unique + Expect(lo.Uniq(tempInstanceTypeOrder)).To(HaveLen(len(tempInstanceTypeOrder))) + + // We have to lock since we are doing simultaneous access to this value + mu.Lock() + if len(instanceTypeOrder) == 0 { + instanceTypeOrder = tempInstanceTypeOrder + } else { + Expect(tempInstanceTypeOrder).To(BeEquivalentTo(instanceTypeOrder)) + } + mu.Unlock() + }() + } + wg.Wait() + }) }) func uniqueInstanceTypeList(instanceTypesLists [][]*corecloudprovider.InstanceType) { @@ -2272,10 +2479,10 @@ func uniqueInstanceTypeList(instanceTypesLists [][]*corecloudprovider.InstanceTy // generateSpotPricing creates a spot price history output for use in a mock that has all spot offerings discounted by 50% // vs the on-demand offering. -func generateSpotPricing(cp *cloudprovider.CloudProvider, nodePool *corev1beta1.NodePool) *ec2.DescribeSpotPriceHistoryOutput { +func generateSpotPricing(cp *cloudprovider.CloudProvider, nodePool *karpv1.NodePool) *ec2.DescribeSpotPriceHistoryOutput { rsp := &ec2.DescribeSpotPriceHistoryOutput{} instanceTypes, err := cp.GetInstanceTypes(ctx, nodePool) - awsEnv.InstanceTypeCache.Flush() + awsEnv.InstanceTypesProvider.Reset() Expect(err).To(Succeed()) t := fakeClock.Now() @@ -2283,18 +2490,19 @@ func generateSpotPricing(cp *cloudprovider.CloudProvider, nodePool *corev1beta1. instanceType := it onDemandPrice := 1.00 for _, o := range it.Offerings { - if o.CapacityType == corev1beta1.CapacityTypeOnDemand { + if o.Requirements.Get(karpv1.CapacityTypeLabelKey).Any() == karpv1.CapacityTypeOnDemand { onDemandPrice = o.Price } } for _, o := range instanceType.Offerings { o := o - if o.CapacityType != corev1beta1.CapacityTypeSpot { + if o.Requirements.Get(karpv1.CapacityTypeLabelKey).Any() != karpv1.CapacityTypeSpot { continue } + zone := o.Requirements.Get(corev1.LabelTopologyZone).Any() spotPrice := fmt.Sprintf("%0.3f", onDemandPrice*0.5) rsp.SpotPriceHistory = append(rsp.SpotPriceHistory, &ec2.SpotPrice{ - AvailabilityZone: &o.Zone, + AvailabilityZone: &zone, InstanceType: &instanceType.Name, SpotPrice: &spotPrice, Timestamp: &t, diff --git a/pkg/providers/instancetype/types.go b/pkg/providers/instancetype/types.go index 6cebec2f523b..e19d31363686 100644 --- a/pkg/providers/instancetype/types.go +++ b/pkg/providers/instancetype/types.go @@ -25,13 +25,12 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "knative.dev/pkg/ptr" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" @@ -50,7 +49,7 @@ var ( ) func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, region string, - blockDeviceMappings []*v1beta1.BlockDeviceMapping, instanceStorePolicy *v1beta1.InstanceStorePolicy, maxPods *int32, podsPerCore *int32, + blockDeviceMappings []*v1.BlockDeviceMapping, instanceStorePolicy *v1.InstanceStorePolicy, maxPods *int32, podsPerCore *int32, kubeReserved map[string]string, systemReserved map[string]string, evictionHard map[string]string, evictionSoft map[string]string, amiFamily amifamily.AMIFamily, offerings cloudprovider.Offerings) *cloudprovider.InstanceType { @@ -65,8 +64,8 @@ func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, region str EvictionThreshold: evictionThreshold(memory(ctx, info), ephemeralStorage(info, amiFamily, blockDeviceMappings, instanceStorePolicy), amiFamily, evictionHard, evictionSoft), }, } - if it.Requirements.Compatible(scheduling.NewRequirements(scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, string(v1.Windows)))) == nil { - it.Capacity[v1beta1.ResourcePrivateIPv4Address] = *privateIPv4Address(info) + if it.Requirements.Compatible(scheduling.NewRequirements(scheduling.NewRequirement(corev1.LabelOSStable, corev1.NodeSelectorOpIn, string(corev1.Windows)))) == nil { + it.Capacity[v1.ResourcePrivateIPv4Address] = *privateIPv4Address(aws.StringValue(info.InstanceType)) } return it } @@ -75,99 +74,116 @@ func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, region str func computeRequirements(info *ec2.InstanceTypeInfo, offerings cloudprovider.Offerings, region string, amiFamily amifamily.AMIFamily) scheduling.Requirements { requirements := scheduling.NewRequirements( // Well Known Upstream - scheduling.NewRequirement(v1.LabelInstanceTypeStable, v1.NodeSelectorOpIn, aws.StringValue(info.InstanceType)), - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, getArchitecture(info)), - scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, getOS(info, amiFamily)...), - scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, lo.Map(offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.Zone })...), - scheduling.NewRequirement(v1.LabelTopologyRegion, v1.NodeSelectorOpIn, region), - scheduling.NewRequirement(v1.LabelWindowsBuild, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(corev1.LabelInstanceTypeStable, corev1.NodeSelectorOpIn, aws.StringValue(info.InstanceType)), + scheduling.NewRequirement(corev1.LabelArchStable, corev1.NodeSelectorOpIn, getArchitecture(info)), + scheduling.NewRequirement(corev1.LabelOSStable, corev1.NodeSelectorOpIn, getOS(info, amiFamily)...), + scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, lo.Map(offerings.Available(), func(o cloudprovider.Offering, _ int) string { + return o.Requirements.Get(corev1.LabelTopologyZone).Any() + })...), + scheduling.NewRequirement(corev1.LabelTopologyRegion, corev1.NodeSelectorOpIn, region), + scheduling.NewRequirement(corev1.LabelWindowsBuild, corev1.NodeSelectorOpDoesNotExist), // Well Known to Karpenter - scheduling.NewRequirement(corev1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, lo.Map(offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.CapacityType })...), + scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, lo.Map(offerings.Available(), func(o cloudprovider.Offering, _ int) string { + return o.Requirements.Get(karpv1.CapacityTypeLabelKey).Any() + })...), // Well Known to AWS - scheduling.NewRequirement(v1beta1.LabelInstanceCPU, v1.NodeSelectorOpIn, fmt.Sprint(aws.Int64Value(info.VCpuInfo.DefaultVCpus))), - scheduling.NewRequirement(v1beta1.LabelInstanceCPUManufacturer, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceMemory, v1.NodeSelectorOpIn, fmt.Sprint(aws.Int64Value(info.MemoryInfo.SizeInMiB))), - scheduling.NewRequirement(v1beta1.LabelInstanceNetworkBandwidth, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceCategory, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceFamily, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceGeneration, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceLocalNVME, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceSize, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceGPUName, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceGPUManufacturer, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceGPUMemory, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorName, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorManufacturer, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1beta1.LabelInstanceHypervisor, v1.NodeSelectorOpIn, aws.StringValue(info.Hypervisor)), - scheduling.NewRequirement(v1beta1.LabelInstanceEncryptionInTransitSupported, v1.NodeSelectorOpIn, fmt.Sprint(aws.BoolValue(info.NetworkInfo.EncryptionInTransitSupported))), + scheduling.NewRequirement(v1.LabelInstanceCPU, corev1.NodeSelectorOpIn, fmt.Sprint(aws.Int64Value(info.VCpuInfo.DefaultVCpus))), + scheduling.NewRequirement(v1.LabelInstanceCPUManufacturer, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceMemory, corev1.NodeSelectorOpIn, fmt.Sprint(aws.Int64Value(info.MemoryInfo.SizeInMiB))), + scheduling.NewRequirement(v1.LabelInstanceEBSBandwidth, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceNetworkBandwidth, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceCategory, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceFamily, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceGeneration, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceLocalNVME, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceSize, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceGPUName, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceGPUManufacturer, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceGPUCount, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceGPUMemory, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceAcceleratorName, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceAcceleratorManufacturer, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceAcceleratorCount, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceHypervisor, corev1.NodeSelectorOpIn, aws.StringValue(info.Hypervisor)), + scheduling.NewRequirement(v1.LabelInstanceEncryptionInTransitSupported, corev1.NodeSelectorOpIn, fmt.Sprint(aws.BoolValue(info.NetworkInfo.EncryptionInTransitSupported))), ) + // Only add zone-id label when available in offerings. It may not be available if a user has upgraded from a + // previous version of Karpenter w/o zone-id support and the nodeclass subnet status has not yet updated. + if zoneIDs := lo.FilterMap(offerings.Available(), func(o cloudprovider.Offering, _ int) (string, bool) { + zoneID := o.Requirements.Get(v1.LabelTopologyZoneID).Any() + return zoneID, zoneID != "" + }); len(zoneIDs) != 0 { + requirements.Add(scheduling.NewRequirement(v1.LabelTopologyZoneID, corev1.NodeSelectorOpIn, zoneIDs...)) + } // Instance Type Labels instanceFamilyParts := instanceTypeScheme.FindStringSubmatch(aws.StringValue(info.InstanceType)) if len(instanceFamilyParts) == 4 { - requirements[v1beta1.LabelInstanceCategory].Insert(instanceFamilyParts[1]) - requirements[v1beta1.LabelInstanceGeneration].Insert(instanceFamilyParts[3]) + requirements[v1.LabelInstanceCategory].Insert(instanceFamilyParts[1]) + requirements[v1.LabelInstanceGeneration].Insert(instanceFamilyParts[3]) } instanceTypeParts := strings.Split(aws.StringValue(info.InstanceType), ".") if len(instanceTypeParts) == 2 { - requirements.Get(v1beta1.LabelInstanceFamily).Insert(instanceTypeParts[0]) - requirements.Get(v1beta1.LabelInstanceSize).Insert(instanceTypeParts[1]) + requirements.Get(v1.LabelInstanceFamily).Insert(instanceTypeParts[0]) + requirements.Get(v1.LabelInstanceSize).Insert(instanceTypeParts[1]) } if info.InstanceStorageInfo != nil && aws.StringValue(info.InstanceStorageInfo.NvmeSupport) != ec2.EphemeralNvmeSupportUnsupported { - requirements[v1beta1.LabelInstanceLocalNVME].Insert(fmt.Sprint(aws.Int64Value(info.InstanceStorageInfo.TotalSizeInGB))) + requirements[v1.LabelInstanceLocalNVME].Insert(fmt.Sprint(aws.Int64Value(info.InstanceStorageInfo.TotalSizeInGB))) } // Network bandwidth if bandwidth, ok := InstanceTypeBandwidthMegabits[aws.StringValue(info.InstanceType)]; ok { - requirements[v1beta1.LabelInstanceNetworkBandwidth].Insert(fmt.Sprint(bandwidth)) + requirements[v1.LabelInstanceNetworkBandwidth].Insert(fmt.Sprint(bandwidth)) } // GPU Labels if info.GpuInfo != nil && len(info.GpuInfo.Gpus) == 1 { gpu := info.GpuInfo.Gpus[0] - requirements.Get(v1beta1.LabelInstanceGPUName).Insert(lowerKabobCase(aws.StringValue(gpu.Name))) - requirements.Get(v1beta1.LabelInstanceGPUManufacturer).Insert(lowerKabobCase(aws.StringValue(gpu.Manufacturer))) - requirements.Get(v1beta1.LabelInstanceGPUCount).Insert(fmt.Sprint(aws.Int64Value(gpu.Count))) - requirements.Get(v1beta1.LabelInstanceGPUMemory).Insert(fmt.Sprint(aws.Int64Value(gpu.MemoryInfo.SizeInMiB))) + requirements.Get(v1.LabelInstanceGPUName).Insert(lowerKabobCase(aws.StringValue(gpu.Name))) + requirements.Get(v1.LabelInstanceGPUManufacturer).Insert(lowerKabobCase(aws.StringValue(gpu.Manufacturer))) + requirements.Get(v1.LabelInstanceGPUCount).Insert(fmt.Sprint(aws.Int64Value(gpu.Count))) + requirements.Get(v1.LabelInstanceGPUMemory).Insert(fmt.Sprint(aws.Int64Value(gpu.MemoryInfo.SizeInMiB))) } // Accelerators if info.InferenceAcceleratorInfo != nil && len(info.InferenceAcceleratorInfo.Accelerators) == 1 { accelerator := info.InferenceAcceleratorInfo.Accelerators[0] - requirements.Get(v1beta1.LabelInstanceAcceleratorName).Insert(lowerKabobCase(aws.StringValue(accelerator.Name))) - requirements.Get(v1beta1.LabelInstanceAcceleratorManufacturer).Insert(lowerKabobCase(aws.StringValue(accelerator.Manufacturer))) - requirements.Get(v1beta1.LabelInstanceAcceleratorCount).Insert(fmt.Sprint(aws.Int64Value(accelerator.Count))) + requirements.Get(v1.LabelInstanceAcceleratorName).Insert(lowerKabobCase(aws.StringValue(accelerator.Name))) + requirements.Get(v1.LabelInstanceAcceleratorManufacturer).Insert(lowerKabobCase(aws.StringValue(accelerator.Manufacturer))) + requirements.Get(v1.LabelInstanceAcceleratorCount).Insert(fmt.Sprint(aws.Int64Value(accelerator.Count))) } // Windows Build Version Labels if family, ok := amiFamily.(*amifamily.Windows); ok { - requirements.Get(v1.LabelWindowsBuild).Insert(family.Build) + requirements.Get(corev1.LabelWindowsBuild).Insert(family.Build) } // Trn1 Accelerators // TODO: remove function once DescribeInstanceTypes contains the accelerator data // Values found from: https://aws.amazon.com/ec2/instance-types/trn1/ if strings.HasPrefix(*info.InstanceType, "trn1") { - requirements.Get(v1beta1.LabelInstanceAcceleratorName).Insert(lowerKabobCase("Inferentia")) - requirements.Get(v1beta1.LabelInstanceAcceleratorManufacturer).Insert(lowerKabobCase("AWS")) - requirements.Get(v1beta1.LabelInstanceAcceleratorCount).Insert(fmt.Sprint(awsNeurons(info))) + requirements.Get(v1.LabelInstanceAcceleratorName).Insert(lowerKabobCase("Inferentia")) + requirements.Get(v1.LabelInstanceAcceleratorManufacturer).Insert(lowerKabobCase("AWS")) + requirements.Get(v1.LabelInstanceAcceleratorCount).Insert(fmt.Sprint(awsNeurons(info))) } // CPU Manufacturer, valid options: aws, intel, amd if info.ProcessorInfo != nil { - requirements.Get(v1beta1.LabelInstanceCPUManufacturer).Insert(lowerKabobCase(aws.StringValue(info.ProcessorInfo.Manufacturer))) + requirements.Get(v1.LabelInstanceCPUManufacturer).Insert(lowerKabobCase(aws.StringValue(info.ProcessorInfo.Manufacturer))) + } + // EBS Max Bandwidth + if info.EbsInfo != nil && info.EbsInfo.EbsOptimizedInfo != nil && aws.StringValue(info.EbsInfo.EbsOptimizedSupport) == ec2.EbsOptimizedSupportDefault { + requirements.Get(v1.LabelInstanceEBSBandwidth).Insert(fmt.Sprint(aws.Int64Value(info.EbsInfo.EbsOptimizedInfo.MaximumBandwidthInMbps))) } return requirements } func getOS(info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily) []string { if _, ok := amiFamily.(*amifamily.Windows); ok { - if getArchitecture(info) == corev1beta1.ArchitectureAmd64 { - return []string{string(v1.Windows)} + if getArchitecture(info) == karpv1.ArchitectureAmd64 { + return []string{string(corev1.Windows)} } return []string{} } - return []string{string(v1.Linux)} + return []string{string(corev1.Linux)} } func getArchitecture(info *ec2.InstanceTypeInfo) string { for _, architecture := range info.ProcessorInfo.SupportedArchitectures { - if value, ok := v1beta1.AWSToKubeArchitectures[aws.StringValue(architecture)]; ok { + if value, ok := v1.AWSToKubeArchitectures[aws.StringValue(architecture)]; ok { return value } } @@ -175,20 +191,20 @@ func getArchitecture(info *ec2.InstanceTypeInfo) string { } func computeCapacity(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, - blockDeviceMapping []*v1beta1.BlockDeviceMapping, instanceStorePolicy *v1beta1.InstanceStorePolicy, - maxPods *int32, podsPerCore *int32) v1.ResourceList { - - resourceList := v1.ResourceList{ - v1.ResourceCPU: *cpu(info), - v1.ResourceMemory: *memory(ctx, info), - v1.ResourceEphemeralStorage: *ephemeralStorage(info, amiFamily, blockDeviceMapping, instanceStorePolicy), - v1.ResourcePods: *pods(ctx, info, amiFamily, maxPods, podsPerCore), - v1beta1.ResourceAWSPodENI: *awsPodENI(aws.StringValue(info.InstanceType)), - v1beta1.ResourceNVIDIAGPU: *nvidiaGPUs(info), - v1beta1.ResourceAMDGPU: *amdGPUs(info), - v1beta1.ResourceAWSNeuron: *awsNeurons(info), - v1beta1.ResourceHabanaGaudi: *habanaGaudis(info), - v1beta1.ResourceEFA: *efas(info), + blockDeviceMapping []*v1.BlockDeviceMapping, instanceStorePolicy *v1.InstanceStorePolicy, + maxPods *int32, podsPerCore *int32) corev1.ResourceList { + + resourceList := corev1.ResourceList{ + corev1.ResourceCPU: *cpu(info), + corev1.ResourceMemory: *memory(ctx, info), + corev1.ResourceEphemeralStorage: *ephemeralStorage(info, amiFamily, blockDeviceMapping, instanceStorePolicy), + corev1.ResourcePods: *pods(ctx, info, amiFamily, maxPods, podsPerCore), + v1.ResourceAWSPodENI: *awsPodENI(aws.StringValue(info.InstanceType)), + v1.ResourceNVIDIAGPU: *nvidiaGPUs(info), + v1.ResourceAMDGPU: *amdGPUs(info), + v1.ResourceAWSNeuron: *awsNeurons(info), + v1.ResourceHabanaGaudi: *habanaGaudis(info), + v1.ResourceEFA: *efas(info), } return resourceList } @@ -210,16 +226,16 @@ func memory(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Quantity } // Setting ephemeral-storage to be either the default value, what is defined in blockDeviceMappings, or the combined size of local store volumes. -func ephemeralStorage(info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, blockDeviceMappings []*v1beta1.BlockDeviceMapping, instanceStorePolicy *v1beta1.InstanceStorePolicy) *resource.Quantity { +func ephemeralStorage(info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, blockDeviceMappings []*v1.BlockDeviceMapping, instanceStorePolicy *v1.InstanceStorePolicy) *resource.Quantity { // If local store disks have been configured for node ephemeral-storage, use the total size of the disks. - if lo.FromPtr(instanceStorePolicy) == v1beta1.InstanceStorePolicyRAID0 { + if lo.FromPtr(instanceStorePolicy) == v1.InstanceStorePolicyRAID0 { if info.InstanceStorageInfo != nil && info.InstanceStorageInfo.TotalSizeInGB != nil { return resources.Quantity(fmt.Sprintf("%dG", *info.InstanceStorageInfo.TotalSizeInGB)) } } if len(blockDeviceMappings) != 0 { // First check if there's a root volume configured in blockDeviceMappings. - if blockDeviceMapping, ok := lo.Find(blockDeviceMappings, func(bdm *v1beta1.BlockDeviceMapping) bool { + if blockDeviceMapping, ok := lo.Find(blockDeviceMappings, func(bdm *v1.BlockDeviceMapping) bool { return bdm.RootVolume }); ok && blockDeviceMapping.EBS.VolumeSize != nil { return blockDeviceMapping.EBS.VolumeSize @@ -231,7 +247,7 @@ func ephemeralStorage(info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, return lo.Ternary(volumeSize != nil, volumeSize, amifamily.DefaultEBS.VolumeSize) default: // If a block device mapping exists in the provider for the root volume, use the volume size specified in the provider. If not, use the default - if blockDeviceMapping, ok := lo.Find(blockDeviceMappings, func(bdm *v1beta1.BlockDeviceMapping) bool { + if blockDeviceMapping, ok := lo.Find(blockDeviceMappings, func(bdm *v1.BlockDeviceMapping) bool { return *bdm.DeviceName == *amiFamily.EphemeralBlockDevice() }); ok && blockDeviceMapping.EBS.VolumeSize != nil { return blockDeviceMapping.EBS.VolumeSize @@ -239,7 +255,7 @@ func ephemeralStorage(info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, } } //Return the ephemeralBlockDevice size if defined in ami - if ephemeralBlockDevice, ok := lo.Find(amiFamily.DefaultBlockDeviceMappings(), func(item *v1beta1.BlockDeviceMapping) bool { + if ephemeralBlockDevice, ok := lo.Find(amiFamily.DefaultBlockDeviceMappings(), func(item *v1.BlockDeviceMapping) bool { return *amiFamily.EphemeralBlockDevice() == *item.DeviceName }); ok { return ephemeralBlockDevice.EBS.VolumeSize @@ -247,9 +263,10 @@ func ephemeralStorage(info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, return amifamily.DefaultEBS.VolumeSize } -func awsPodENI(name string) *resource.Quantity { +// awsPodENI relies on the VPC resource controller to populate the vpc.amazonaws.com/pod-eni resource +func awsPodENI(instanceTypeName string) *resource.Quantity { // https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html#supported-instance-types - limits, ok := Limits[name] + limits, ok := Limits[instanceTypeName] if ok && limits.IsTrunkingCompatible { return resources.Quantity(fmt.Sprint(limits.BranchInterface)) } @@ -321,12 +338,12 @@ func efas(info *ec2.InstanceTypeInfo) *resource.Quantity { func ENILimitedPods(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Quantity { // The number of pods per node is calculated using the formula: // max number of ENIs * (IPv4 Addresses per ENI -1) + 2 - // https://github.com/awslabs/amazon-eks-ami/blob/master/files/eni-max-pods.txt#L20 + // https://github.com/awslabs/amazon-eks-ami/blob/main/templates/shared/runtime/eni-max-pods.txt // VPC CNI only uses the default network interface // https://github.com/aws/amazon-vpc-cni-k8s/blob/3294231c0dce52cfe473bf6c62f47956a3b333b6/scripts/gen_vpc_ip_limits.go#L162 networkInterfaces := *info.NetworkInfo.NetworkCards[*info.NetworkInfo.DefaultNetworkCardIndex].MaximumNetworkInterfaces - usableNetworkInterfaces := lo.Max([]int64{(networkInterfaces - int64(options.FromContext(ctx).ReservedENIs)), 0}) + usableNetworkInterfaces := lo.Max([]int64{networkInterfaces - int64(options.FromContext(ctx).ReservedENIs), 0}) if usableNetworkInterfaces == 0 { return resource.NewQuantity(0, resource.DecimalSI) } @@ -334,25 +351,28 @@ func ENILimitedPods(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Q return resources.Quantity(fmt.Sprint(usableNetworkInterfaces*(addressesPerInterface-1) + 2)) } -func privateIPv4Address(info *ec2.InstanceTypeInfo) *resource.Quantity { +func privateIPv4Address(instanceTypeName string) *resource.Quantity { //https://github.com/aws/amazon-vpc-resource-controller-k8s/blob/ecbd6965a0100d9a070110233762593b16023287/pkg/provider/ip/provider.go#L297 - capacity := aws.Int64Value(info.NetworkInfo.Ipv4AddressesPerInterface) - 1 - return resources.Quantity(fmt.Sprint(capacity)) + limits, ok := Limits[instanceTypeName] + if !ok { + return resources.Quantity("0") + } + return resources.Quantity(fmt.Sprint(limits.IPv4PerInterface - 1)) } -func systemReservedResources(systemReserved map[string]string) v1.ResourceList { - return lo.MapEntries(systemReserved, func(k string, v string) (v1.ResourceName, resource.Quantity) { - return v1.ResourceName(k), resource.MustParse(v) +func systemReservedResources(systemReserved map[string]string) corev1.ResourceList { + return lo.MapEntries(systemReserved, func(k string, v string) (corev1.ResourceName, resource.Quantity) { + return corev1.ResourceName(k), resource.MustParse(v) }) } -func kubeReservedResources(cpus, pods, eniLimitedPods *resource.Quantity, amiFamily amifamily.AMIFamily, kubeReserved map[string]string) v1.ResourceList { +func kubeReservedResources(cpus, pods, eniLimitedPods *resource.Quantity, amiFamily amifamily.AMIFamily, kubeReserved map[string]string) corev1.ResourceList { if amiFamily.FeatureFlags().UsesENILimitedMemoryOverhead { pods = eniLimitedPods } - resources := v1.ResourceList{ - v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dMi", (11*pods.Value())+255)), - v1.ResourceEphemeralStorage: resource.MustParse("1Gi"), // default kube-reserved ephemeral-storage + resources := corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dMi", (11*pods.Value())+255)), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), // default kube-reserved ephemeral-storage } // kube-reserved Computed from // https://github.com/bottlerocket-os/bottlerocket/pull/1388/files#diff-bba9e4e3e46203be2b12f22e0d654ebd270f0b478dd34f40c31d7aa695620f2fR611 @@ -373,21 +393,21 @@ func kubeReservedResources(cpus, pods, eniLimitedPods *resource.Quantity, amiFam } cpuOverhead := resources.Cpu() cpuOverhead.Add(*resource.NewMilliQuantity(int64(r*cpuRange.percentage), resource.DecimalSI)) - resources[v1.ResourceCPU] = *cpuOverhead + resources[corev1.ResourceCPU] = *cpuOverhead } } - return lo.Assign(resources, lo.MapEntries(kubeReserved, func(k string, v string) (v1.ResourceName, resource.Quantity) { - return v1.ResourceName(k), resource.MustParse(v) + return lo.Assign(resources, lo.MapEntries(kubeReserved, func(k string, v string) (corev1.ResourceName, resource.Quantity) { + return corev1.ResourceName(k), resource.MustParse(v) })) } -func evictionThreshold(memory *resource.Quantity, storage *resource.Quantity, amiFamily amifamily.AMIFamily, evictionHard map[string]string, evictionSoft map[string]string) v1.ResourceList { - overhead := v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("100Mi"), - v1.ResourceEphemeralStorage: resource.MustParse(fmt.Sprint(math.Ceil(float64(storage.Value()) / 100 * 10))), +func evictionThreshold(memory *resource.Quantity, storage *resource.Quantity, amiFamily amifamily.AMIFamily, evictionHard map[string]string, evictionSoft map[string]string) corev1.ResourceList { + overhead := corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("100Mi"), + corev1.ResourceEphemeralStorage: resource.MustParse(fmt.Sprint(math.Ceil(float64(storage.Value()) / 100 * 10))), } - override := v1.ResourceList{} + override := corev1.ResourceList{} var evictionSignals []map[string]string if evictionHard != nil { evictionSignals = append(evictionSignals, evictionHard) @@ -397,12 +417,12 @@ func evictionThreshold(memory *resource.Quantity, storage *resource.Quantity, am } for _, m := range evictionSignals { - temp := v1.ResourceList{} + temp := corev1.ResourceList{} if v, ok := m[MemoryAvailable]; ok { - temp[v1.ResourceMemory] = computeEvictionSignal(*memory, v) + temp[corev1.ResourceMemory] = computeEvictionSignal(*memory, v) } if v, ok := m[NodeFSAvailable]; ok { - temp[v1.ResourceEphemeralStorage] = computeEvictionSignal(*storage, v) + temp[corev1.ResourceEphemeralStorage] = computeEvictionSignal(*storage, v) } override = resources.MaxResources(override, temp) } @@ -414,15 +434,15 @@ func pods(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.A var count int64 switch { case maxPods != nil: - count = int64(ptr.Int32Value(maxPods)) + count = int64(lo.FromPtr(maxPods)) case amiFamily.FeatureFlags().SupportsENILimitedPodDensity: count = ENILimitedPods(ctx, info).Value() default: count = 110 } - if ptr.Int32Value(podsPerCore) > 0 && amiFamily.FeatureFlags().PodsPerCoreEnabled { - count = lo.Min([]int64{int64(ptr.Int32Value(podsPerCore)) * ptr.Int64Value(info.VCpuInfo.DefaultVCpus), count}) + if lo.FromPtr(podsPerCore) > 0 && amiFamily.FeatureFlags().PodsPerCoreEnabled { + count = lo.Min([]int64{int64(lo.FromPtr(podsPerCore)) * lo.FromPtr(info.VCpuInfo.DefaultVCpus), count}) } return resources.Quantity(fmt.Sprint(count)) } diff --git a/pkg/providers/instancetype/zz_generated.bandwidth.go b/pkg/providers/instancetype/zz_generated.bandwidth.go index f8f92d72731f..d0f8cbbc83ed 100644 --- a/pkg/providers/instancetype/zz_generated.bandwidth.go +++ b/pkg/providers/instancetype/zz_generated.bandwidth.go @@ -20,745 +20,816 @@ package instancetype var ( InstanceTypeBandwidthMegabits = map[string]int64{ - // c3.large is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html - // c4.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html - // i2.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html - // m2.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html - // m4.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html - // r3.large is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html - // t1.micro is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html - // t2.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html - "t3.nano": 32, - "t3a.nano": 32, - "t4g.nano": 32, - "t3.micro": 64, - "t3a.micro": 64, - "t4g.micro": 64, - "t3.small": 128, - "t3a.small": 128, - "t4g.small": 128, - "t3.medium": 256, - "t3a.medium": 256, - "t4g.medium": 256, - "c7a.medium": 390, - "m7a.medium": 390, - "m7i-flex.large": 390, - "r7a.medium": 390, - "a1.medium": 500, - "c6g.medium": 500, - "c6gd.medium": 500, - "m6g.medium": 500, - "m6gd.medium": 500, - "r6g.medium": 500, - "r6gd.medium": 500, - "x2gd.medium": 500, - "t3.large": 512, - "t3a.large": 512, - "t4g.large": 512, - "c7g.medium": 520, - "c7gd.medium": 520, - "m7g.medium": 520, - "m7gd.medium": 520, - "r7g.medium": 520, - "r7gd.medium": 520, - "x1e.xlarge": 625, - "a1.large": 750, - "c5.large": 750, - "c5a.large": 750, - "c5ad.large": 750, - "c5d.large": 750, - "c6g.large": 750, - "c6gd.large": 750, - "i3.large": 750, - "m5.large": 750, - "m5a.large": 750, - "m5ad.large": 750, - "m5d.large": 750, - "m6g.large": 750, - "m6gd.large": 750, - "r4.large": 750, - "r5.large": 750, - "r5a.large": 750, - "r5ad.large": 750, - "r5b.large": 750, - "r5d.large": 750, - "r6g.large": 750, - "r6gd.large": 750, - "x2gd.large": 750, - "z1d.large": 750, - "c6a.large": 781, - "c6i.large": 781, - "c6id.large": 781, - "c7a.large": 781, - "c7i.large": 781, - "i4g.large": 781, - "i4i.large": 781, - "m6a.large": 781, - "m6i.large": 781, - "m6id.large": 781, - "m7a.large": 781, - "m7i-flex.xlarge": 781, - "m7i.large": 781, - "r6a.large": 781, - "r6i.large": 781, - "r6id.large": 781, - "r7a.large": 781, - "r7i.large": 781, - "r7iz.large": 781, - "c7g.large": 937, - "c7gd.large": 937, - "m7g.large": 937, - "m7gd.large": 937, - "r7g.large": 937, - "r7gd.large": 937, - "t3.xlarge": 1024, - "t3a.xlarge": 1024, - "t4g.xlarge": 1024, - "a1.xlarge": 1250, - "c5.xlarge": 1250, - "c5a.xlarge": 1250, - "c5ad.xlarge": 1250, - "c5d.xlarge": 1250, - "c6g.xlarge": 1250, - "c6gd.xlarge": 1250, - "g5g.xlarge": 1250, - "i3.xlarge": 1250, - "m5.xlarge": 1250, - "m5a.xlarge": 1250, - "m5ad.xlarge": 1250, - "m5d.xlarge": 1250, - "m6g.xlarge": 1250, - "m6gd.xlarge": 1250, - "r4.xlarge": 1250, - "r5.xlarge": 1250, - "r5a.xlarge": 1250, - "r5ad.xlarge": 1250, - "r5b.xlarge": 1250, - "r5d.xlarge": 1250, - "r6g.xlarge": 1250, - "r6gd.xlarge": 1250, - "x1e.2xlarge": 1250, - "x2gd.xlarge": 1250, - "z1d.xlarge": 1250, - "c6a.xlarge": 1562, - "c6i.xlarge": 1562, - "c6id.xlarge": 1562, - "c7a.xlarge": 1562, - "c7i.xlarge": 1562, - "is4gen.medium": 1562, - "m6a.xlarge": 1562, - "m6i.xlarge": 1562, - "m6id.xlarge": 1562, - "m7a.xlarge": 1562, - "m7i-flex.2xlarge": 1562, - "m7i.xlarge": 1562, - "r6a.xlarge": 1562, - "r6i.xlarge": 1562, - "r6id.xlarge": 1562, - "r7a.xlarge": 1562, - "r7i.xlarge": 1562, - "r7iz.xlarge": 1562, - "c6gn.medium": 1600, - "i4g.xlarge": 1875, - "i4i.xlarge": 1875, - "x2iedn.xlarge": 1875, - "c7g.xlarge": 1876, - "c7gd.xlarge": 1876, - "m7g.xlarge": 1876, - "m7gd.xlarge": 1876, - "r7g.xlarge": 1876, - "r7gd.xlarge": 1876, - "g4ad.xlarge": 2000, - "t3.2xlarge": 2048, - "t3a.2xlarge": 2048, - "t4g.2xlarge": 2048, - "inf2.xlarge": 2083, - "i3en.large": 2100, - "m5dn.large": 2100, - "m5n.large": 2100, - "r5dn.large": 2100, - "r5n.large": 2100, - "a1.2xlarge": 2500, - "c5.2xlarge": 2500, - "c5a.2xlarge": 2500, - "c5ad.2xlarge": 2500, - "c5d.2xlarge": 2500, - "c6g.2xlarge": 2500, - "c6gd.2xlarge": 2500, - "f1.2xlarge": 2500, - "g5.xlarge": 2500, - "g5g.2xlarge": 2500, - "h1.2xlarge": 2500, - "i3.2xlarge": 2500, - "m5.2xlarge": 2500, - "m5a.2xlarge": 2500, - "m5ad.2xlarge": 2500, - "m5d.2xlarge": 2500, - "m6g.2xlarge": 2500, - "m6gd.2xlarge": 2500, - "r4.2xlarge": 2500, - "r5.2xlarge": 2500, - "r5a.2xlarge": 2500, - "r5ad.2xlarge": 2500, - "r5b.2xlarge": 2500, - "r5d.2xlarge": 2500, - "r6g.2xlarge": 2500, - "r6gd.2xlarge": 2500, - "x1e.4xlarge": 2500, - "x2gd.2xlarge": 2500, - "z1d.2xlarge": 2500, - "c5n.large": 3000, - "c6gn.large": 3000, - "d3.xlarge": 3000, - "m5zn.large": 3000, - "vt1.3xlarge": 3120, - "c6a.2xlarge": 3125, - "c6i.2xlarge": 3125, - "c6id.2xlarge": 3125, - "c6in.large": 3125, - "c7a.2xlarge": 3125, - "c7gn.medium": 3125, - "c7i.2xlarge": 3125, - "im4gn.large": 3125, - "is4gen.large": 3125, - "m6a.2xlarge": 3125, - "m6i.2xlarge": 3125, - "m6id.2xlarge": 3125, - "m6idn.large": 3125, - "m6in.large": 3125, - "m7a.2xlarge": 3125, - "m7i-flex.4xlarge": 3125, - "m7i.2xlarge": 3125, - "r6a.2xlarge": 3125, - "r6i.2xlarge": 3125, - "r6id.2xlarge": 3125, - "r6idn.large": 3125, - "r6in.large": 3125, - "r7a.2xlarge": 3125, - "r7i.2xlarge": 3125, - "r7iz.2xlarge": 3125, - "trn1.2xlarge": 3125, - "c7g.2xlarge": 3750, - "c7gd.2xlarge": 3750, - "m7g.2xlarge": 3750, - "m7gd.2xlarge": 3750, - "r7g.2xlarge": 3750, - "r7gd.2xlarge": 3750, - "m5dn.xlarge": 4100, - "m5n.xlarge": 4100, - "r5dn.xlarge": 4100, - "r5n.xlarge": 4100, - "g4ad.2xlarge": 4167, - "i3en.xlarge": 4200, - "i4g.2xlarge": 4687, - "i4i.2xlarge": 4687, - "a1.4xlarge": 5000, - "a1.metal": 5000, - "c5.4xlarge": 5000, - "c5a.4xlarge": 5000, - "c5ad.4xlarge": 5000, - "c5d.4xlarge": 5000, - "c5n.xlarge": 5000, - "c6g.4xlarge": 5000, - "c6gd.4xlarge": 5000, - "f1.4xlarge": 5000, - "g3.4xlarge": 5000, - "g4dn.xlarge": 5000, - "g5.2xlarge": 5000, - "g5g.4xlarge": 5000, - "h1.4xlarge": 5000, - "i3.4xlarge": 5000, - "inf1.2xlarge": 5000, - "inf1.xlarge": 5000, - "m5.4xlarge": 5000, - "m5a.4xlarge": 5000, - "m5ad.4xlarge": 5000, - "m5d.4xlarge": 5000, - "m5zn.xlarge": 5000, - "m6g.4xlarge": 5000, - "m6gd.4xlarge": 5000, - "r4.4xlarge": 5000, - "r5.4xlarge": 5000, - "r5a.4xlarge": 5000, - "r5ad.4xlarge": 5000, - "r5b.4xlarge": 5000, - "r5d.4xlarge": 5000, - "r6g.4xlarge": 5000, - "r6gd.4xlarge": 5000, - "x1e.8xlarge": 5000, - "x2gd.4xlarge": 5000, - "x2iedn.2xlarge": 5000, - "z1d.3xlarge": 5000, - "d3.2xlarge": 6000, - "d3en.xlarge": 6000, - "c6a.4xlarge": 6250, - "c6i.4xlarge": 6250, - "c6id.4xlarge": 6250, - "c6in.xlarge": 6250, - "c7a.4xlarge": 6250, - "c7gn.large": 6250, - "c7i.4xlarge": 6250, - "im4gn.xlarge": 6250, - "is4gen.xlarge": 6250, - "m6a.4xlarge": 6250, - "m6i.4xlarge": 6250, - "m6id.4xlarge": 6250, - "m6idn.xlarge": 6250, - "m6in.xlarge": 6250, - "m7a.4xlarge": 6250, - "m7i-flex.8xlarge": 6250, - "m7i.4xlarge": 6250, - "r6a.4xlarge": 6250, - "r6i.4xlarge": 6250, - "r6id.4xlarge": 6250, - "r6idn.xlarge": 6250, - "r6in.xlarge": 6250, - "r7a.4xlarge": 6250, - "r7i.4xlarge": 6250, - "r7iz.4xlarge": 6250, - "vt1.6xlarge": 6250, - "c6gn.xlarge": 6300, - "c7g.4xlarge": 7500, - "c7gd.4xlarge": 7500, - "m5a.8xlarge": 7500, - "m5ad.8xlarge": 7500, - "m7g.4xlarge": 7500, - "m7gd.4xlarge": 7500, - "r5a.8xlarge": 7500, - "r5ad.8xlarge": 7500, - "r7g.4xlarge": 7500, - "r7gd.4xlarge": 7500, - "m5dn.2xlarge": 8125, - "m5n.2xlarge": 8125, - "r5dn.2xlarge": 8125, - "r5n.2xlarge": 8125, - "g4ad.4xlarge": 8333, - "i3en.2xlarge": 8400, - "i4g.4xlarge": 9375, - "i4i.4xlarge": 9375, - "c3.8xlarge": 10000, - "c4.8xlarge": 10000, - "c5a.8xlarge": 10000, - "c5ad.8xlarge": 10000, - "c5n.2xlarge": 10000, - "d2.8xlarge": 10000, - "g3.8xlarge": 10000, - "g4dn.2xlarge": 10000, - "g5.4xlarge": 10000, - "h1.8xlarge": 10000, - "i2.8xlarge": 10000, - "i3.8xlarge": 10000, - "m4.10xlarge": 10000, - "m5.8xlarge": 10000, - "m5a.12xlarge": 10000, - "m5ad.12xlarge": 10000, - "m5d.8xlarge": 10000, - "m5zn.2xlarge": 10000, - "mac2-m2.metal": 10000, - "mac2-m2pro.metal": 10000, - "mac2.metal": 10000, - "p2.8xlarge": 10000, - "p3.8xlarge": 10000, - "r3.8xlarge": 10000, - "r4.8xlarge": 10000, - "r5.8xlarge": 10000, - "r5a.12xlarge": 10000, - "r5ad.12xlarge": 10000, - "r5b.8xlarge": 10000, - "r5d.8xlarge": 10000, - "x1.16xlarge": 10000, - "x1e.16xlarge": 10000, - "c5.12xlarge": 12000, - "c5.9xlarge": 12000, - "c5a.12xlarge": 12000, - "c5ad.12xlarge": 12000, - "c5d.12xlarge": 12000, - "c5d.9xlarge": 12000, - "c6g.8xlarge": 12000, - "c6gd.8xlarge": 12000, - "g5g.8xlarge": 12000, - "m5.12xlarge": 12000, - "m5a.16xlarge": 12000, - "m5ad.16xlarge": 12000, - "m5d.12xlarge": 12000, - "m6g.8xlarge": 12000, - "m6gd.8xlarge": 12000, - "r5.12xlarge": 12000, - "r5a.16xlarge": 12000, - "r5ad.16xlarge": 12000, - "r5b.12xlarge": 12000, - "r5d.12xlarge": 12000, - "r6g.8xlarge": 12000, - "r6gd.8xlarge": 12000, - "x2gd.8xlarge": 12000, - "z1d.6xlarge": 12000, - "c6a.8xlarge": 12500, - "c6gn.2xlarge": 12500, - "c6i.8xlarge": 12500, - "c6id.8xlarge": 12500, - "c6in.2xlarge": 12500, - "c7a.8xlarge": 12500, - "c7gn.xlarge": 12500, - "c7i.8xlarge": 12500, - "d3.4xlarge": 12500, - "d3en.2xlarge": 12500, - "i3en.3xlarge": 12500, - "im4gn.2xlarge": 12500, - "is4gen.2xlarge": 12500, - "m6a.8xlarge": 12500, - "m6i.8xlarge": 12500, - "m6id.8xlarge": 12500, - "m6idn.2xlarge": 12500, - "m6in.2xlarge": 12500, - "m7a.8xlarge": 12500, - "m7i.8xlarge": 12500, - "r6a.8xlarge": 12500, - "r6i.8xlarge": 12500, - "r6id.8xlarge": 12500, - "r6idn.2xlarge": 12500, - "r6in.2xlarge": 12500, - "r7a.8xlarge": 12500, - "r7i.8xlarge": 12500, - "r7iz.8xlarge": 12500, - "x2iedn.4xlarge": 12500, - "x2iezn.2xlarge": 12500, - "c5n.4xlarge": 15000, - "c7g.8xlarge": 15000, - "c7gd.8xlarge": 15000, - "g4ad.8xlarge": 15000, - "m5zn.3xlarge": 15000, - "m7g.8xlarge": 15000, - "m7gd.8xlarge": 15000, - "r7g.8xlarge": 15000, - "r7gd.8xlarge": 15000, - "x2iezn.4xlarge": 15000, - "m5dn.4xlarge": 16250, - "m5n.4xlarge": 16250, - "r5dn.4xlarge": 16250, - "r5n.4xlarge": 16250, - "inf2.8xlarge": 16667, - "c6a.12xlarge": 18750, - "c6i.12xlarge": 18750, - "c6id.12xlarge": 18750, - "c7a.12xlarge": 18750, - "c7i.12xlarge": 18750, - "i4g.8xlarge": 18750, - "i4i.8xlarge": 18750, - "m6a.12xlarge": 18750, - "m6i.12xlarge": 18750, - "m6id.12xlarge": 18750, - "m7a.12xlarge": 18750, - "m7i.12xlarge": 18750, - "r6a.12xlarge": 18750, - "r6i.12xlarge": 18750, - "r6id.12xlarge": 18750, - "r7a.12xlarge": 18750, - "r7i.12xlarge": 18750, - "c5a.16xlarge": 20000, - "c5a.24xlarge": 20000, - "c5ad.16xlarge": 20000, - "c5ad.24xlarge": 20000, - "c6g.12xlarge": 20000, - "c6gd.12xlarge": 20000, - "g4dn.4xlarge": 20000, - "m5.16xlarge": 20000, - "m5a.24xlarge": 20000, - "m5ad.24xlarge": 20000, - "m5d.16xlarge": 20000, - "m6g.12xlarge": 20000, - "m6gd.12xlarge": 20000, - "r5.16xlarge": 20000, - "r5a.24xlarge": 20000, - "r5ad.24xlarge": 20000, - "r5b.16xlarge": 20000, - "r5d.16xlarge": 20000, - "r6g.12xlarge": 20000, - "r6gd.12xlarge": 20000, - "x2gd.12xlarge": 20000, - "c7g.12xlarge": 22500, - "c7gd.12xlarge": 22500, - "m7g.12xlarge": 22500, - "m7gd.12xlarge": 22500, - "r7g.12xlarge": 22500, - "r7gd.12xlarge": 22500, - "c5.18xlarge": 25000, - "c5.24xlarge": 25000, - "c5.metal": 25000, - "c5d.18xlarge": 25000, - "c5d.24xlarge": 25000, - "c5d.metal": 25000, - "c6a.16xlarge": 25000, - "c6g.16xlarge": 25000, - "c6g.metal": 25000, - "c6gd.16xlarge": 25000, - "c6gd.metal": 25000, - "c6gn.4xlarge": 25000, - "c6i.16xlarge": 25000, - "c6id.16xlarge": 25000, - "c6in.4xlarge": 25000, - "c7a.16xlarge": 25000, - "c7gn.2xlarge": 25000, - "c7i.16xlarge": 25000, - "d3.8xlarge": 25000, - "d3en.4xlarge": 25000, - "f1.16xlarge": 25000, - "g3.16xlarge": 25000, - "g4ad.16xlarge": 25000, - "g5.16xlarge": 25000, - "g5.8xlarge": 25000, - "g5g.16xlarge": 25000, - "g5g.metal": 25000, - "h1.16xlarge": 25000, - "i3.16xlarge": 25000, - "i3.metal": 25000, - "i3en.6xlarge": 25000, - "im4gn.4xlarge": 25000, - "inf1.6xlarge": 25000, - "is4gen.4xlarge": 25000, - "m4.16xlarge": 25000, - "m5.24xlarge": 25000, - "m5.metal": 25000, - "m5d.24xlarge": 25000, - "m5d.metal": 25000, - "m5dn.8xlarge": 25000, - "m5n.8xlarge": 25000, - "m6a.16xlarge": 25000, - "m6g.16xlarge": 25000, - "m6g.metal": 25000, - "m6gd.16xlarge": 25000, - "m6gd.metal": 25000, - "m6i.16xlarge": 25000, - "m6id.16xlarge": 25000, - "m6idn.4xlarge": 25000, - "m6in.4xlarge": 25000, - "m7a.16xlarge": 25000, - "m7i.16xlarge": 25000, - "mac1.metal": 25000, - "p2.16xlarge": 25000, - "p3.16xlarge": 25000, - "r4.16xlarge": 25000, - "r5.24xlarge": 25000, - "r5.metal": 25000, - "r5b.24xlarge": 25000, - "r5b.metal": 25000, - "r5d.24xlarge": 25000, - "r5d.metal": 25000, - "r5dn.8xlarge": 25000, - "r5n.8xlarge": 25000, - "r6a.16xlarge": 25000, - "r6g.16xlarge": 25000, - "r6g.metal": 25000, - "r6gd.16xlarge": 25000, - "r6gd.metal": 25000, - "r6i.16xlarge": 25000, - "r6id.16xlarge": 25000, - "r6idn.4xlarge": 25000, - "r6in.4xlarge": 25000, - "r7a.16xlarge": 25000, - "r7i.16xlarge": 25000, - "r7iz.12xlarge": 25000, - "r7iz.16xlarge": 25000, - "r7iz.metal-16xl": 25000, - "vt1.24xlarge": 25000, - "x1.32xlarge": 25000, - "x1e.32xlarge": 25000, - "x2gd.16xlarge": 25000, - "x2gd.metal": 25000, - "x2iedn.8xlarge": 25000, - "z1d.12xlarge": 25000, - "z1d.metal": 25000, - "i4i.12xlarge": 28120, - "c7g.16xlarge": 30000, - "c7g.metal": 30000, - "c7gd.16xlarge": 30000, - "c7gd.metal": 30000, - "m7g.16xlarge": 30000, - "m7g.metal": 30000, - "m7gd.16xlarge": 30000, - "m7gd.metal": 30000, - "r7g.16xlarge": 30000, - "r7g.metal": 30000, - "r7gd.16xlarge": 30000, - "r7gd.metal": 30000, - "c6a.24xlarge": 37500, - "c6i.24xlarge": 37500, - "c6id.24xlarge": 37500, - "c7a.24xlarge": 37500, - "c7i.24xlarge": 37500, - "c7i.metal-24xl": 37500, - "i4g.16xlarge": 37500, - "i4i.16xlarge": 37500, - "m6a.24xlarge": 37500, - "m6i.24xlarge": 37500, - "m6id.24xlarge": 37500, - "m7a.24xlarge": 37500, - "m7i.24xlarge": 37500, - "m7i.metal-24xl": 37500, - "r6a.24xlarge": 37500, - "r6i.24xlarge": 37500, - "r6id.24xlarge": 37500, - "r7a.24xlarge": 37500, - "r7i.24xlarge": 37500, - "r7i.metal-24xl": 37500, - "d3en.6xlarge": 40000, - "g5.12xlarge": 40000, - "c5n.9xlarge": 50000, - "c6a.32xlarge": 50000, - "c6a.48xlarge": 50000, - "c6a.metal": 50000, - "c6gn.8xlarge": 50000, - "c6i.32xlarge": 50000, - "c6i.metal": 50000, - "c6id.32xlarge": 50000, - "c6id.metal": 50000, - "c6in.8xlarge": 50000, - "c7a.32xlarge": 50000, - "c7a.48xlarge": 50000, - "c7a.metal-48xl": 50000, - "c7gn.4xlarge": 50000, - "c7i.48xlarge": 50000, - "c7i.metal-48xl": 50000, - "d3en.8xlarge": 50000, - "g4dn.12xlarge": 50000, - "g4dn.16xlarge": 50000, - "g4dn.8xlarge": 50000, - "g5.24xlarge": 50000, - "i3en.12xlarge": 50000, - "im4gn.8xlarge": 50000, - "inf2.24xlarge": 50000, - "is4gen.8xlarge": 50000, - "m5dn.12xlarge": 50000, - "m5n.12xlarge": 50000, - "m5zn.6xlarge": 50000, - "m6a.32xlarge": 50000, - "m6a.48xlarge": 50000, - "m6a.metal": 50000, - "m6i.32xlarge": 50000, - "m6i.metal": 50000, - "m6id.32xlarge": 50000, - "m6id.metal": 50000, - "m6idn.8xlarge": 50000, - "m6in.8xlarge": 50000, - "m7a.32xlarge": 50000, - "m7a.48xlarge": 50000, - "m7a.metal-48xl": 50000, - "m7i.48xlarge": 50000, - "m7i.metal-48xl": 50000, - "r5dn.12xlarge": 50000, - "r5n.12xlarge": 50000, - "r6a.32xlarge": 50000, - "r6a.48xlarge": 50000, - "r6a.metal": 50000, - "r6i.32xlarge": 50000, - "r6i.metal": 50000, - "r6id.32xlarge": 50000, - "r6id.metal": 50000, - "r6idn.8xlarge": 50000, - "r6in.8xlarge": 50000, - "r7a.32xlarge": 50000, - "r7a.48xlarge": 50000, - "r7a.metal-48xl": 50000, - "r7i.48xlarge": 50000, - "r7i.metal-48xl": 50000, - "r7iz.32xlarge": 50000, - "r7iz.metal-32xl": 50000, - "u-3tb1.56xlarge": 50000, - "x2idn.16xlarge": 50000, - "x2iedn.16xlarge": 50000, - "x2iezn.6xlarge": 50000, - "i4i.24xlarge": 56250, - "c6gn.12xlarge": 75000, - "c6in.12xlarge": 75000, - "d3en.12xlarge": 75000, - "i4i.32xlarge": 75000, - "i4i.metal": 75000, - "m5dn.16xlarge": 75000, - "m5n.16xlarge": 75000, - "m6idn.12xlarge": 75000, - "m6in.12xlarge": 75000, - "r5dn.16xlarge": 75000, - "r5n.16xlarge": 75000, - "r6idn.12xlarge": 75000, - "r6in.12xlarge": 75000, - "x2idn.24xlarge": 75000, - "x2iedn.24xlarge": 75000, - "x2iezn.8xlarge": 75000, - "c5n.18xlarge": 100000, - "c5n.metal": 100000, - "c6gn.16xlarge": 100000, - "c6in.16xlarge": 100000, - "c7gn.8xlarge": 100000, - "dl2q.24xlarge": 100000, - "g4dn.metal": 100000, - "g5.48xlarge": 100000, - "hpc6a.48xlarge": 100000, - "i3en.24xlarge": 100000, - "i3en.metal": 100000, - "im4gn.16xlarge": 100000, - "inf1.24xlarge": 100000, - "inf2.48xlarge": 100000, - "m5dn.24xlarge": 100000, - "m5dn.metal": 100000, - "m5n.24xlarge": 100000, - "m5n.metal": 100000, - "m5zn.12xlarge": 100000, - "m5zn.metal": 100000, - "m6idn.16xlarge": 100000, - "m6in.16xlarge": 100000, - "p3dn.24xlarge": 100000, - "r5dn.24xlarge": 100000, - "r5dn.metal": 100000, - "r5n.24xlarge": 100000, - "r5n.metal": 100000, - "r6idn.16xlarge": 100000, - "r6in.16xlarge": 100000, - "u-12tb1.112xlarge": 100000, - "u-12tb1.metal": 100000, - "u-18tb1.112xlarge": 100000, - "u-18tb1.metal": 100000, - "u-24tb1.112xlarge": 100000, - "u-24tb1.metal": 100000, - "u-6tb1.112xlarge": 100000, - "u-6tb1.56xlarge": 100000, - "u-6tb1.metal": 100000, - "u-9tb1.112xlarge": 100000, - "u-9tb1.metal": 100000, - "x2idn.32xlarge": 100000, - "x2idn.metal": 100000, - "x2iedn.32xlarge": 100000, - "x2iedn.metal": 100000, - "x2iezn.12xlarge": 100000, - "x2iezn.metal": 100000, - "c6in.24xlarge": 150000, - "c7gn.12xlarge": 150000, - "m6idn.24xlarge": 150000, - "m6in.24xlarge": 150000, - "r6idn.24xlarge": 150000, - "r6in.24xlarge": 150000, - "c6in.32xlarge": 200000, - "c6in.metal": 200000, - "c7gn.16xlarge": 200000, - "hpc6id.32xlarge": 200000, - "hpc7g.16xlarge": 200000, - "hpc7g.4xlarge": 200000, - "hpc7g.8xlarge": 200000, - "m6idn.32xlarge": 200000, - "m6idn.metal": 200000, - "m6in.32xlarge": 200000, - "m6in.metal": 200000, - "r6idn.32xlarge": 200000, - "r6idn.metal": 200000, - "r6in.32xlarge": 200000, - "r6in.metal": 200000, - "hpc7a.12xlarge": 300000, - "hpc7a.24xlarge": 300000, - "hpc7a.48xlarge": 300000, - "hpc7a.96xlarge": 300000, - "dl1.24xlarge": 400000, - "p4d.24xlarge": 400000, - "p4de.24xlarge": 400000, - "trn1.32xlarge": 800000, - "trn1n.32xlarge": 1600000, - "p5.48xlarge": 3200000, + // c1.medium has vague bandwidth information, bandwidth is Moderate + // c1.xlarge has vague bandwidth information, bandwidth is High + // c3.2xlarge has vague bandwidth information, bandwidth is High + // c3.4xlarge has vague bandwidth information, bandwidth is High + // c3.large has vague bandwidth information, bandwidth is Moderate + // c3.xlarge has vague bandwidth information, bandwidth is Moderate + // c4.2xlarge has vague bandwidth information, bandwidth is High + // c4.4xlarge has vague bandwidth information, bandwidth is High + // c4.large has vague bandwidth information, bandwidth is Moderate + // c4.xlarge has vague bandwidth information, bandwidth is High + // d2.2xlarge has vague bandwidth information, bandwidth is High + // d2.4xlarge has vague bandwidth information, bandwidth is High + // d2.xlarge has vague bandwidth information, bandwidth is Moderate + // f1.2xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit + // f1.4xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit + // g3.4xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit + // g3s.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html + // i2.2xlarge has vague bandwidth information, bandwidth is High + // i2.4xlarge has vague bandwidth information, bandwidth is High + // i2.xlarge has vague bandwidth information, bandwidth is Moderate + // m1.large has vague bandwidth information, bandwidth is Moderate + // m1.medium has vague bandwidth information, bandwidth is Moderate + // m1.small has vague bandwidth information, bandwidth is Low + // m1.xlarge has vague bandwidth information, bandwidth is High + // m2.2xlarge has vague bandwidth information, bandwidth is Moderate + // m2.4xlarge has vague bandwidth information, bandwidth is High + // m2.xlarge has vague bandwidth information, bandwidth is Moderate + // m3.2xlarge has vague bandwidth information, bandwidth is High + // m3.large has vague bandwidth information, bandwidth is Moderate + // m3.medium has vague bandwidth information, bandwidth is Moderate + // m3.xlarge has vague bandwidth information, bandwidth is High + // m4.2xlarge has vague bandwidth information, bandwidth is High + // m4.4xlarge has vague bandwidth information, bandwidth is High + // m4.large has vague bandwidth information, bandwidth is Moderate + // m4.xlarge has vague bandwidth information, bandwidth is High + // p2.xlarge has vague bandwidth information, bandwidth is High + // p3.2xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit + // r3.2xlarge has vague bandwidth information, bandwidth is High + // r3.4xlarge has vague bandwidth information, bandwidth is High + // r3.large has vague bandwidth information, bandwidth is Moderate + // r3.xlarge has vague bandwidth information, bandwidth is Moderate + // t1.micro has vague bandwidth information, bandwidth is Very Low + // t2.2xlarge has vague bandwidth information, bandwidth is Moderate + // t2.large has vague bandwidth information, bandwidth is Low to Moderate + // t2.medium has vague bandwidth information, bandwidth is Low to Moderate + // t2.micro has vague bandwidth information, bandwidth is Low to Moderate + // t2.nano has vague bandwidth information, bandwidth is Low to Moderate + // t2.small has vague bandwidth information, bandwidth is Low to Moderate + // t2.xlarge has vague bandwidth information, bandwidth is Moderate + "t3.nano": 32, + "t3a.nano": 32, + "t4g.nano": 32, + "t3.micro": 64, + "t3a.micro": 64, + "t4g.micro": 64, + "t3.small": 128, + "t3a.small": 128, + "t4g.small": 128, + "t3.medium": 256, + "t3a.medium": 256, + "t4g.medium": 256, + "c7a.medium": 390, + "c7i-flex.large": 390, + "m7a.medium": 390, + "m7i-flex.large": 390, + "r7a.medium": 390, + "a1.medium": 500, + "c6g.medium": 500, + "c6gd.medium": 500, + "m6g.medium": 500, + "m6gd.medium": 500, + "r6g.medium": 500, + "r6gd.medium": 500, + "x2gd.medium": 500, + "t3.large": 512, + "t3a.large": 512, + "t4g.large": 512, + "c7g.medium": 520, + "c7gd.medium": 520, + "m7g.medium": 520, + "m7gd.medium": 520, + "r7g.medium": 520, + "r7gd.medium": 520, + "r8g.medium": 520, + "x1e.xlarge": 625, + "a1.large": 750, + "c5.large": 750, + "c5a.large": 750, + "c5ad.large": 750, + "c5d.large": 750, + "c6g.large": 750, + "c6gd.large": 750, + "i3.large": 750, + "m5.large": 750, + "m5a.large": 750, + "m5ad.large": 750, + "m5d.large": 750, + "m6g.large": 750, + "m6gd.large": 750, + "r4.large": 750, + "r5.large": 750, + "r5a.large": 750, + "r5ad.large": 750, + "r5b.large": 750, + "r5d.large": 750, + "r6g.large": 750, + "r6gd.large": 750, + "x2gd.large": 750, + "z1d.large": 750, + "c6a.large": 781, + "c6i.large": 781, + "c6id.large": 781, + "c7a.large": 781, + "c7i-flex.xlarge": 781, + "c7i.large": 781, + "i4g.large": 781, + "i4i.large": 781, + "m6a.large": 781, + "m6i.large": 781, + "m6id.large": 781, + "m7a.large": 781, + "m7i-flex.xlarge": 781, + "m7i.large": 781, + "r6a.large": 781, + "r6i.large": 781, + "r6id.large": 781, + "r7a.large": 781, + "r7i.large": 781, + "r7iz.large": 781, + "c7g.large": 937, + "c7gd.large": 937, + "m7g.large": 937, + "m7gd.large": 937, + "r7g.large": 937, + "r7gd.large": 937, + "r8g.large": 937, + "t3.xlarge": 1024, + "t3a.xlarge": 1024, + "t4g.xlarge": 1024, + "a1.xlarge": 1250, + "c5.xlarge": 1250, + "c5a.xlarge": 1250, + "c5ad.xlarge": 1250, + "c5d.xlarge": 1250, + "c6g.xlarge": 1250, + "c6gd.xlarge": 1250, + "g5g.xlarge": 1250, + "i3.xlarge": 1250, + "m5.xlarge": 1250, + "m5a.xlarge": 1250, + "m5ad.xlarge": 1250, + "m5d.xlarge": 1250, + "m6g.xlarge": 1250, + "m6gd.xlarge": 1250, + "r4.xlarge": 1250, + "r5.xlarge": 1250, + "r5a.xlarge": 1250, + "r5ad.xlarge": 1250, + "r5b.xlarge": 1250, + "r5d.xlarge": 1250, + "r6g.xlarge": 1250, + "r6gd.xlarge": 1250, + "x1e.2xlarge": 1250, + "x2gd.xlarge": 1250, + "z1d.xlarge": 1250, + "c6a.xlarge": 1562, + "c6i.xlarge": 1562, + "c6id.xlarge": 1562, + "c7a.xlarge": 1562, + "c7i-flex.2xlarge": 1562, + "c7i.xlarge": 1562, + "is4gen.medium": 1562, + "m6a.xlarge": 1562, + "m6i.xlarge": 1562, + "m6id.xlarge": 1562, + "m7a.xlarge": 1562, + "m7i-flex.2xlarge": 1562, + "m7i.xlarge": 1562, + "r6a.xlarge": 1562, + "r6i.xlarge": 1562, + "r6id.xlarge": 1562, + "r7a.xlarge": 1562, + "r7i.xlarge": 1562, + "r7iz.xlarge": 1562, + "c6gn.medium": 1600, + "i4g.xlarge": 1875, + "i4i.xlarge": 1875, + "x2iedn.xlarge": 1875, + "c7g.xlarge": 1876, + "c7gd.xlarge": 1876, + "m7g.xlarge": 1876, + "m7gd.xlarge": 1876, + "r7g.xlarge": 1876, + "r7gd.xlarge": 1876, + "r8g.xlarge": 1876, + "g4ad.xlarge": 2000, + "t3.2xlarge": 2048, + "t3a.2xlarge": 2048, + "t4g.2xlarge": 2048, + "inf2.xlarge": 2083, + "i3en.large": 2100, + "m5dn.large": 2100, + "m5n.large": 2100, + "r5dn.large": 2100, + "r5n.large": 2100, + "a1.2xlarge": 2500, + "c5.2xlarge": 2500, + "c5a.2xlarge": 2500, + "c5ad.2xlarge": 2500, + "c5d.2xlarge": 2500, + "c6g.2xlarge": 2500, + "c6gd.2xlarge": 2500, + "g5.xlarge": 2500, + "g5g.2xlarge": 2500, + "g6.xlarge": 2500, + "h1.2xlarge": 2500, + "i3.2xlarge": 2500, + "m5.2xlarge": 2500, + "m5a.2xlarge": 2500, + "m5ad.2xlarge": 2500, + "m5d.2xlarge": 2500, + "m6g.2xlarge": 2500, + "m6gd.2xlarge": 2500, + "r4.2xlarge": 2500, + "r5.2xlarge": 2500, + "r5a.2xlarge": 2500, + "r5ad.2xlarge": 2500, + "r5b.2xlarge": 2500, + "r5d.2xlarge": 2500, + "r6g.2xlarge": 2500, + "r6gd.2xlarge": 2500, + "x1e.4xlarge": 2500, + "x2gd.2xlarge": 2500, + "z1d.2xlarge": 2500, + "c5n.large": 3000, + "c6gn.large": 3000, + "d3.xlarge": 3000, + "m5zn.large": 3000, + "vt1.3xlarge": 3120, + "c6a.2xlarge": 3125, + "c6i.2xlarge": 3125, + "c6id.2xlarge": 3125, + "c6in.large": 3125, + "c7a.2xlarge": 3125, + "c7gn.medium": 3125, + "c7i-flex.4xlarge": 3125, + "c7i.2xlarge": 3125, + "im4gn.large": 3125, + "is4gen.large": 3125, + "m6a.2xlarge": 3125, + "m6i.2xlarge": 3125, + "m6id.2xlarge": 3125, + "m6idn.large": 3125, + "m6in.large": 3125, + "m7a.2xlarge": 3125, + "m7i-flex.4xlarge": 3125, + "m7i.2xlarge": 3125, + "r6a.2xlarge": 3125, + "r6i.2xlarge": 3125, + "r6id.2xlarge": 3125, + "r6idn.large": 3125, + "r6in.large": 3125, + "r7a.2xlarge": 3125, + "r7i.2xlarge": 3125, + "r7iz.2xlarge": 3125, + "trn1.2xlarge": 3125, + "c7g.2xlarge": 3750, + "c7gd.2xlarge": 3750, + "m7g.2xlarge": 3750, + "m7gd.2xlarge": 3750, + "r7g.2xlarge": 3750, + "r7gd.2xlarge": 3750, + "r8g.2xlarge": 3750, + "m5dn.xlarge": 4100, + "m5n.xlarge": 4100, + "r5dn.xlarge": 4100, + "r5n.xlarge": 4100, + "g4ad.2xlarge": 4167, + "i3en.xlarge": 4200, + "i4g.2xlarge": 4687, + "i4i.2xlarge": 4687, + "a1.4xlarge": 5000, + "a1.metal": 5000, + "c5.4xlarge": 5000, + "c5a.4xlarge": 5000, + "c5ad.4xlarge": 5000, + "c5d.4xlarge": 5000, + "c5n.xlarge": 5000, + "c6g.4xlarge": 5000, + "c6gd.4xlarge": 5000, + "g4dn.xlarge": 5000, + "g5.2xlarge": 5000, + "g5g.4xlarge": 5000, + "g6.2xlarge": 5000, + "h1.4xlarge": 5000, + "i3.4xlarge": 5000, + "inf1.2xlarge": 5000, + "inf1.xlarge": 5000, + "m5.4xlarge": 5000, + "m5a.4xlarge": 5000, + "m5ad.4xlarge": 5000, + "m5d.4xlarge": 5000, + "m5zn.xlarge": 5000, + "m6g.4xlarge": 5000, + "m6gd.4xlarge": 5000, + "r4.4xlarge": 5000, + "r5.4xlarge": 5000, + "r5a.4xlarge": 5000, + "r5ad.4xlarge": 5000, + "r5b.4xlarge": 5000, + "r5d.4xlarge": 5000, + "r6g.4xlarge": 5000, + "r6gd.4xlarge": 5000, + "x1e.8xlarge": 5000, + "x2gd.4xlarge": 5000, + "x2iedn.2xlarge": 5000, + "z1d.3xlarge": 5000, + "d3.2xlarge": 6000, + "d3en.xlarge": 6000, + "c6a.4xlarge": 6250, + "c6i.4xlarge": 6250, + "c6id.4xlarge": 6250, + "c6in.xlarge": 6250, + "c7a.4xlarge": 6250, + "c7gn.large": 6250, + "c7i-flex.8xlarge": 6250, + "c7i.4xlarge": 6250, + "im4gn.xlarge": 6250, + "is4gen.xlarge": 6250, + "m6a.4xlarge": 6250, + "m6i.4xlarge": 6250, + "m6id.4xlarge": 6250, + "m6idn.xlarge": 6250, + "m6in.xlarge": 6250, + "m7a.4xlarge": 6250, + "m7i-flex.8xlarge": 6250, + "m7i.4xlarge": 6250, + "r6a.4xlarge": 6250, + "r6i.4xlarge": 6250, + "r6id.4xlarge": 6250, + "r6idn.xlarge": 6250, + "r6in.xlarge": 6250, + "r7a.4xlarge": 6250, + "r7i.4xlarge": 6250, + "r7iz.4xlarge": 6250, + "vt1.6xlarge": 6250, + "c6gn.xlarge": 6300, + "c7g.4xlarge": 7500, + "c7gd.4xlarge": 7500, + "m5a.8xlarge": 7500, + "m5ad.8xlarge": 7500, + "m7g.4xlarge": 7500, + "m7gd.4xlarge": 7500, + "r5a.8xlarge": 7500, + "r5ad.8xlarge": 7500, + "r7g.4xlarge": 7500, + "r7gd.4xlarge": 7500, + "r8g.4xlarge": 7500, + "m5dn.2xlarge": 8125, + "m5n.2xlarge": 8125, + "r5dn.2xlarge": 8125, + "r5n.2xlarge": 8125, + "g4ad.4xlarge": 8333, + "i3en.2xlarge": 8400, + "i4g.4xlarge": 9375, + "i4i.4xlarge": 9375, + "c3.8xlarge": 10000, + "c4.8xlarge": 10000, + "c5a.8xlarge": 10000, + "c5ad.8xlarge": 10000, + "c5n.2xlarge": 10000, + "d2.8xlarge": 10000, + "g3.8xlarge": 10000, + "g4dn.2xlarge": 10000, + "g5.4xlarge": 10000, + "g6.4xlarge": 10000, + "gr6.4xlarge": 10000, + "h1.8xlarge": 10000, + "i2.8xlarge": 10000, + "i3.8xlarge": 10000, + "m4.10xlarge": 10000, + "m5.8xlarge": 10000, + "m5a.12xlarge": 10000, + "m5ad.12xlarge": 10000, + "m5d.8xlarge": 10000, + "m5zn.2xlarge": 10000, + "mac2-m1ultra.metal": 10000, + "mac2-m2.metal": 10000, + "mac2-m2pro.metal": 10000, + "mac2.metal": 10000, + "p2.8xlarge": 10000, + "p3.8xlarge": 10000, + "r3.8xlarge": 10000, + "r4.8xlarge": 10000, + "r5.8xlarge": 10000, + "r5a.12xlarge": 10000, + "r5ad.12xlarge": 10000, + "r5b.8xlarge": 10000, + "r5d.8xlarge": 10000, + "x1.16xlarge": 10000, + "x1e.16xlarge": 10000, + "c5.12xlarge": 12000, + "c5.9xlarge": 12000, + "c5a.12xlarge": 12000, + "c5ad.12xlarge": 12000, + "c5d.12xlarge": 12000, + "c5d.9xlarge": 12000, + "c6g.8xlarge": 12000, + "c6gd.8xlarge": 12000, + "g5g.8xlarge": 12000, + "m5.12xlarge": 12000, + "m5a.16xlarge": 12000, + "m5ad.16xlarge": 12000, + "m5d.12xlarge": 12000, + "m6g.8xlarge": 12000, + "m6gd.8xlarge": 12000, + "r5.12xlarge": 12000, + "r5a.16xlarge": 12000, + "r5ad.16xlarge": 12000, + "r5b.12xlarge": 12000, + "r5d.12xlarge": 12000, + "r6g.8xlarge": 12000, + "r6gd.8xlarge": 12000, + "x2gd.8xlarge": 12000, + "z1d.6xlarge": 12000, + "c6a.8xlarge": 12500, + "c6gn.2xlarge": 12500, + "c6i.8xlarge": 12500, + "c6id.8xlarge": 12500, + "c6in.2xlarge": 12500, + "c7a.8xlarge": 12500, + "c7gn.xlarge": 12500, + "c7i.8xlarge": 12500, + "d3.4xlarge": 12500, + "d3en.2xlarge": 12500, + "i3en.3xlarge": 12500, + "im4gn.2xlarge": 12500, + "is4gen.2xlarge": 12500, + "m6a.8xlarge": 12500, + "m6i.8xlarge": 12500, + "m6id.8xlarge": 12500, + "m6idn.2xlarge": 12500, + "m6in.2xlarge": 12500, + "m7a.8xlarge": 12500, + "m7i.8xlarge": 12500, + "r6a.8xlarge": 12500, + "r6i.8xlarge": 12500, + "r6id.8xlarge": 12500, + "r6idn.2xlarge": 12500, + "r6in.2xlarge": 12500, + "r7a.8xlarge": 12500, + "r7i.8xlarge": 12500, + "r7iz.8xlarge": 12500, + "x2iedn.4xlarge": 12500, + "x2iezn.2xlarge": 12500, + "c5n.4xlarge": 15000, + "c7g.8xlarge": 15000, + "c7gd.8xlarge": 15000, + "g4ad.8xlarge": 15000, + "m5zn.3xlarge": 15000, + "m7g.8xlarge": 15000, + "m7gd.8xlarge": 15000, + "r7g.8xlarge": 15000, + "r7gd.8xlarge": 15000, + "r8g.8xlarge": 15000, + "x2iezn.4xlarge": 15000, + "m5dn.4xlarge": 16250, + "m5n.4xlarge": 16250, + "r5dn.4xlarge": 16250, + "r5n.4xlarge": 16250, + "inf2.8xlarge": 16667, + "c6a.12xlarge": 18750, + "c6i.12xlarge": 18750, + "c6id.12xlarge": 18750, + "c7a.12xlarge": 18750, + "c7i.12xlarge": 18750, + "i4g.8xlarge": 18750, + "i4i.8xlarge": 18750, + "m6a.12xlarge": 18750, + "m6i.12xlarge": 18750, + "m6id.12xlarge": 18750, + "m7a.12xlarge": 18750, + "m7i.12xlarge": 18750, + "r6a.12xlarge": 18750, + "r6i.12xlarge": 18750, + "r6id.12xlarge": 18750, + "r7a.12xlarge": 18750, + "r7i.12xlarge": 18750, + "c5a.16xlarge": 20000, + "c5a.24xlarge": 20000, + "c5ad.16xlarge": 20000, + "c5ad.24xlarge": 20000, + "c6g.12xlarge": 20000, + "c6gd.12xlarge": 20000, + "g4dn.4xlarge": 20000, + "m5.16xlarge": 20000, + "m5a.24xlarge": 20000, + "m5ad.24xlarge": 20000, + "m5d.16xlarge": 20000, + "m6g.12xlarge": 20000, + "m6gd.12xlarge": 20000, + "r5.16xlarge": 20000, + "r5a.24xlarge": 20000, + "r5ad.24xlarge": 20000, + "r5b.16xlarge": 20000, + "r5d.16xlarge": 20000, + "r6g.12xlarge": 20000, + "r6gd.12xlarge": 20000, + "x2gd.12xlarge": 20000, + "c7g.12xlarge": 22500, + "c7gd.12xlarge": 22500, + "m7g.12xlarge": 22500, + "m7gd.12xlarge": 22500, + "r7g.12xlarge": 22500, + "r7gd.12xlarge": 22500, + "r8g.12xlarge": 22500, + "c5.18xlarge": 25000, + "c5.24xlarge": 25000, + "c5.metal": 25000, + "c5d.18xlarge": 25000, + "c5d.24xlarge": 25000, + "c5d.metal": 25000, + "c6a.16xlarge": 25000, + "c6g.16xlarge": 25000, + "c6g.metal": 25000, + "c6gd.16xlarge": 25000, + "c6gd.metal": 25000, + "c6gn.4xlarge": 25000, + "c6i.16xlarge": 25000, + "c6id.16xlarge": 25000, + "c6in.4xlarge": 25000, + "c7a.16xlarge": 25000, + "c7gn.2xlarge": 25000, + "c7i.16xlarge": 25000, + "d3.8xlarge": 25000, + "d3en.4xlarge": 25000, + "f1.16xlarge": 25000, + "g3.16xlarge": 25000, + "g4ad.16xlarge": 25000, + "g5.16xlarge": 25000, + "g5.8xlarge": 25000, + "g5g.16xlarge": 25000, + "g5g.metal": 25000, + "g6.16xlarge": 25000, + "g6.8xlarge": 25000, + "gr6.8xlarge": 25000, + "h1.16xlarge": 25000, + "i3.16xlarge": 25000, + "i3.metal": 25000, + "i3en.6xlarge": 25000, + "im4gn.4xlarge": 25000, + "inf1.6xlarge": 25000, + "is4gen.4xlarge": 25000, + "m4.16xlarge": 25000, + "m5.24xlarge": 25000, + "m5.metal": 25000, + "m5d.24xlarge": 25000, + "m5d.metal": 25000, + "m5dn.8xlarge": 25000, + "m5n.8xlarge": 25000, + "m6a.16xlarge": 25000, + "m6g.16xlarge": 25000, + "m6g.metal": 25000, + "m6gd.16xlarge": 25000, + "m6gd.metal": 25000, + "m6i.16xlarge": 25000, + "m6id.16xlarge": 25000, + "m6idn.4xlarge": 25000, + "m6in.4xlarge": 25000, + "m7a.16xlarge": 25000, + "m7i.16xlarge": 25000, + "mac1.metal": 25000, + "p2.16xlarge": 25000, + "p3.16xlarge": 25000, + "r4.16xlarge": 25000, + "r5.24xlarge": 25000, + "r5.metal": 25000, + "r5b.24xlarge": 25000, + "r5b.metal": 25000, + "r5d.24xlarge": 25000, + "r5d.metal": 25000, + "r5dn.8xlarge": 25000, + "r5n.8xlarge": 25000, + "r6a.16xlarge": 25000, + "r6g.16xlarge": 25000, + "r6g.metal": 25000, + "r6gd.16xlarge": 25000, + "r6gd.metal": 25000, + "r6i.16xlarge": 25000, + "r6id.16xlarge": 25000, + "r6idn.4xlarge": 25000, + "r6in.4xlarge": 25000, + "r7a.16xlarge": 25000, + "r7i.16xlarge": 25000, + "r7iz.12xlarge": 25000, + "r7iz.16xlarge": 25000, + "r7iz.metal-16xl": 25000, + "vt1.24xlarge": 25000, + "x1.32xlarge": 25000, + "x1e.32xlarge": 25000, + "x2gd.16xlarge": 25000, + "x2gd.metal": 25000, + "x2iedn.8xlarge": 25000, + "z1d.12xlarge": 25000, + "z1d.metal": 25000, + "i4i.12xlarge": 28120, + "c7g.16xlarge": 30000, + "c7g.metal": 30000, + "c7gd.16xlarge": 30000, + "c7gd.metal": 30000, + "m7g.16xlarge": 30000, + "m7g.metal": 30000, + "m7gd.16xlarge": 30000, + "m7gd.metal": 30000, + "r7g.16xlarge": 30000, + "r7g.metal": 30000, + "r7gd.16xlarge": 30000, + "r7gd.metal": 30000, + "r8g.16xlarge": 30000, + "c6a.24xlarge": 37500, + "c6i.24xlarge": 37500, + "c6id.24xlarge": 37500, + "c7a.24xlarge": 37500, + "c7i.24xlarge": 37500, + "c7i.metal-24xl": 37500, + "i4g.16xlarge": 37500, + "i4i.16xlarge": 37500, + "m6a.24xlarge": 37500, + "m6i.24xlarge": 37500, + "m6id.24xlarge": 37500, + "m7a.24xlarge": 37500, + "m7i.24xlarge": 37500, + "m7i.metal-24xl": 37500, + "r6a.24xlarge": 37500, + "r6i.24xlarge": 37500, + "r6id.24xlarge": 37500, + "r7a.24xlarge": 37500, + "r7i.24xlarge": 37500, + "r7i.metal-24xl": 37500, + "d3en.6xlarge": 40000, + "g5.12xlarge": 40000, + "g6.12xlarge": 40000, + "r8g.24xlarge": 40000, + "r8g.metal-24xl": 40000, + "c5n.9xlarge": 50000, + "c6a.32xlarge": 50000, + "c6a.48xlarge": 50000, + "c6a.metal": 50000, + "c6gn.8xlarge": 50000, + "c6i.32xlarge": 50000, + "c6i.metal": 50000, + "c6id.32xlarge": 50000, + "c6id.metal": 50000, + "c6in.8xlarge": 50000, + "c7a.32xlarge": 50000, + "c7a.48xlarge": 50000, + "c7a.metal-48xl": 50000, + "c7gn.4xlarge": 50000, + "c7i.48xlarge": 50000, + "c7i.metal-48xl": 50000, + "d3en.8xlarge": 50000, + "g4dn.12xlarge": 50000, + "g4dn.16xlarge": 50000, + "g4dn.8xlarge": 50000, + "g5.24xlarge": 50000, + "g6.24xlarge": 50000, + "i3en.12xlarge": 50000, + "im4gn.8xlarge": 50000, + "inf2.24xlarge": 50000, + "is4gen.8xlarge": 50000, + "m5dn.12xlarge": 50000, + "m5n.12xlarge": 50000, + "m5zn.6xlarge": 50000, + "m6a.32xlarge": 50000, + "m6a.48xlarge": 50000, + "m6a.metal": 50000, + "m6i.32xlarge": 50000, + "m6i.metal": 50000, + "m6id.32xlarge": 50000, + "m6id.metal": 50000, + "m6idn.8xlarge": 50000, + "m6in.8xlarge": 50000, + "m7a.32xlarge": 50000, + "m7a.48xlarge": 50000, + "m7a.metal-48xl": 50000, + "m7i.48xlarge": 50000, + "m7i.metal-48xl": 50000, + "r5dn.12xlarge": 50000, + "r5n.12xlarge": 50000, + "r6a.32xlarge": 50000, + "r6a.48xlarge": 50000, + "r6a.metal": 50000, + "r6i.32xlarge": 50000, + "r6i.metal": 50000, + "r6id.32xlarge": 50000, + "r6id.metal": 50000, + "r6idn.8xlarge": 50000, + "r6in.8xlarge": 50000, + "r7a.32xlarge": 50000, + "r7a.48xlarge": 50000, + "r7a.metal-48xl": 50000, + "r7i.48xlarge": 50000, + "r7i.metal-48xl": 50000, + "r7iz.32xlarge": 50000, + "r7iz.metal-32xl": 50000, + "r8g.48xlarge": 50000, + "r8g.metal-48xl": 50000, + "u-3tb1.56xlarge": 50000, + "x2idn.16xlarge": 50000, + "x2iedn.16xlarge": 50000, + "x2iezn.6xlarge": 50000, + "i4i.24xlarge": 56250, + "c6gn.12xlarge": 75000, + "c6in.12xlarge": 75000, + "d3en.12xlarge": 75000, + "i4i.32xlarge": 75000, + "i4i.metal": 75000, + "m5dn.16xlarge": 75000, + "m5n.16xlarge": 75000, + "m6idn.12xlarge": 75000, + "m6in.12xlarge": 75000, + "r5dn.16xlarge": 75000, + "r5n.16xlarge": 75000, + "r6idn.12xlarge": 75000, + "r6in.12xlarge": 75000, + "x2idn.24xlarge": 75000, + "x2iedn.24xlarge": 75000, + "x2iezn.8xlarge": 75000, + "c5n.18xlarge": 100000, + "c5n.metal": 100000, + "c6gn.16xlarge": 100000, + "c6in.16xlarge": 100000, + "c7gn.8xlarge": 100000, + "dl2q.24xlarge": 100000, + "g4dn.metal": 100000, + "g5.48xlarge": 100000, + "g6.48xlarge": 100000, + "hpc6a.48xlarge": 100000, + "i3en.24xlarge": 100000, + "i3en.metal": 100000, + "im4gn.16xlarge": 100000, + "inf1.24xlarge": 100000, + "inf2.48xlarge": 100000, + "m5dn.24xlarge": 100000, + "m5dn.metal": 100000, + "m5n.24xlarge": 100000, + "m5n.metal": 100000, + "m5zn.12xlarge": 100000, + "m5zn.metal": 100000, + "m6idn.16xlarge": 100000, + "m6in.16xlarge": 100000, + "p3dn.24xlarge": 100000, + "r5dn.24xlarge": 100000, + "r5dn.metal": 100000, + "r5n.24xlarge": 100000, + "r5n.metal": 100000, + "r6idn.16xlarge": 100000, + "r6in.16xlarge": 100000, + "u-12tb1.112xlarge": 100000, + "u-12tb1.metal": 100000, + "u-18tb1.112xlarge": 100000, + "u-18tb1.metal": 100000, + "u-24tb1.112xlarge": 100000, + "u-24tb1.metal": 100000, + "u-6tb1.112xlarge": 100000, + "u-6tb1.56xlarge": 100000, + "u-6tb1.metal": 100000, + "u-9tb1.112xlarge": 100000, + "u-9tb1.metal": 100000, + "u7i-12tb.224xlarge": 100000, + "x2idn.32xlarge": 100000, + "x2idn.metal": 100000, + "x2iedn.32xlarge": 100000, + "x2iedn.metal": 100000, + "x2iezn.12xlarge": 100000, + "x2iezn.metal": 100000, + "c6in.24xlarge": 150000, + "c7gn.12xlarge": 150000, + "m6idn.24xlarge": 150000, + "m6in.24xlarge": 150000, + "r6idn.24xlarge": 150000, + "r6in.24xlarge": 150000, + "c6in.32xlarge": 200000, + "c6in.metal": 200000, + "c7gn.16xlarge": 200000, + "c7gn.metal": 200000, + "hpc6id.32xlarge": 200000, + "hpc7g.16xlarge": 200000, + "hpc7g.4xlarge": 200000, + "hpc7g.8xlarge": 200000, + "m6idn.32xlarge": 200000, + "m6idn.metal": 200000, + "m6in.32xlarge": 200000, + "m6in.metal": 200000, + "r6idn.32xlarge": 200000, + "r6idn.metal": 200000, + "r6in.32xlarge": 200000, + "r6in.metal": 200000, + "u7in-16tb.224xlarge": 200000, + "u7in-24tb.224xlarge": 200000, + "u7in-32tb.224xlarge": 200000, + "hpc7a.12xlarge": 300000, + "hpc7a.24xlarge": 300000, + "hpc7a.48xlarge": 300000, + "hpc7a.96xlarge": 300000, + "dl1.24xlarge": 400000, + "p4d.24xlarge": 400000, + "p4de.24xlarge": 400000, + "trn1.32xlarge": 800000, + "trn1n.32xlarge": 1600000, + "p5.48xlarge": 3200000, } ) diff --git a/pkg/providers/instancetype/zz_generated.vpclimits.go b/pkg/providers/instancetype/zz_generated.vpclimits.go index da5f712b9b5e..19ab1fc97f11 100644 --- a/pkg/providers/instancetype/zz_generated.vpclimits.go +++ b/pkg/providers/instancetype/zz_generated.vpclimits.go @@ -17,7 +17,7 @@ // so we can get this information at runtime. // Code generated by go generate; DO NOT EDIT. -// This file was generated at 2024-01-29T18:28:02Z +// This file was generated at 2024-04-30T17:56:45Z // WARNING: please add @ellistarn, @bwagner5, or @jonathan-innis from aws/karpenter to reviewers // if you are updating this file since Karpenter is depending on this file to calculate max pods. @@ -1846,19 +1846,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "c6in.32xlarge": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -1911,19 +1911,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "c6in.metal": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -2365,6 +2365,21 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "c7gd.metal": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, "c7gd.xlarge": { Interface: 4, IPv4PerInterface: 15, @@ -2485,6 +2500,21 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "c7gn.metal": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, "c7gn.xlarge": { Interface: 4, IPv4PerInterface: 15, @@ -2983,8 +3013,8 @@ var Limits = map[string]*VPCLimits{ "g3.4xlarge": { Interface: 8, IPv4PerInterface: 30, - IsTrunkingCompatible: false, - BranchInterface: 0, + IsTrunkingCompatible: true, + BranchInterface: 6, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { @@ -2998,8 +3028,8 @@ var Limits = map[string]*VPCLimits{ "g3.8xlarge": { Interface: 8, IPv4PerInterface: 30, - IsTrunkingCompatible: false, - BranchInterface: 0, + IsTrunkingCompatible: true, + BranchInterface: 6, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { @@ -3013,8 +3043,8 @@ var Limits = map[string]*VPCLimits{ "g3s.xlarge": { Interface: 4, IPv4PerInterface: 15, - IsTrunkingCompatible: false, - BranchInterface: 0, + IsTrunkingCompatible: true, + BranchInterface: 10, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { @@ -3415,6 +3445,156 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "g6.12xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 114, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.16xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.24xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.2xlarge": { + Interface: 4, + IPv4PerInterface: 15, + IsTrunkingCompatible: true, + BranchInterface: 38, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 4, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.48xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.4xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 54, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.8xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 84, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.xlarge": { + Interface: 4, + IPv4PerInterface: 15, + IsTrunkingCompatible: true, + BranchInterface: 18, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 4, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "gr6.4xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 54, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "gr6.8xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 84, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, "h1.16xlarge": { Interface: 8, IPv4PerInterface: 50, @@ -6236,19 +6416,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "m6idn.32xlarge": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -6301,19 +6481,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "m6idn.metal": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -6396,19 +6576,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "m6in.32xlarge": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -6461,19 +6641,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "m6in.metal": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -6915,6 +7095,21 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "m7gd.metal": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, "m7gd.xlarge": { Interface: 4, IPv4PerInterface: 15, @@ -9411,19 +9606,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "r6idn.32xlarge": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -9476,19 +9671,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "r6idn.metal": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -9571,19 +9766,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "r6in.32xlarge": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -9636,19 +9831,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "r6in.metal": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -10090,6 +10285,21 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "r7gd.metal": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, "r7gd.xlarge": { Interface: 4, IPv4PerInterface: 15, diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 418228e99ecf..9d72f00344d9 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -16,7 +16,6 @@ package launchtemplate import ( "context" - "errors" "fmt" "math" "net" @@ -26,6 +25,7 @@ import ( "time" "go.uber.org/multierr" + "sigs.k8s.io/controller-runtime/pkg/log" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -35,12 +35,12 @@ import ( "github.com/mitchellh/hashstructure/v2" "github.com/patrickmn/go-cache" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "knative.dev/pkg/logging" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/pkg/apis" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" awserrors "github.com/aws/karpenter-provider-aws/pkg/errors" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" @@ -52,12 +52,10 @@ import ( "sigs.k8s.io/karpenter/pkg/utils/pretty" ) -var karpenterManagedTagKey = fmt.Sprintf("%s/cluster", v1beta1.Group) - type Provider interface { - EnsureAll(context.Context, *v1beta1.EC2NodeClass, *corev1beta1.NodeClaim, + EnsureAll(context.Context, *v1.EC2NodeClass, *karpv1.NodeClaim, []*cloudprovider.InstanceType, string, map[string]string) ([]*LaunchTemplate, error) - DeleteAll(context.Context, *v1beta1.EC2NodeClass) error + DeleteAll(context.Context, *v1.EC2NodeClass) error InvalidateCache(context.Context, string, string) ResolveClusterCIDR(context.Context) error } @@ -111,17 +109,17 @@ func NewDefaultProvider(ctx context.Context, cache *cache.Cache, ec2api ec2iface return l } -func (p *DefaultProvider) EnsureAll(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, +func (p *DefaultProvider) EnsureAll(ctx context.Context, nodeClass *v1.EC2NodeClass, nodeClaim *karpv1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, tags map[string]string) ([]*LaunchTemplate, error) { p.Lock() defer p.Unlock() - options, err := p.createAMIOptions(ctx, nodeClass, lo.Assign(nodeClaim.Labels, map[string]string{corev1beta1.CapacityTypeLabelKey: capacityType}), tags) + options, err := p.createAMIOptions(ctx, nodeClass, lo.Assign(nodeClaim.Labels, map[string]string{karpv1.CapacityTypeLabelKey: capacityType}), tags) if err != nil { return nil, err } - resolvedLaunchTemplates, err := p.amiFamily.Resolve(ctx, nodeClass, nodeClaim, instanceTypes, capacityType, options) + resolvedLaunchTemplates, err := p.amiFamily.Resolve(nodeClass, nodeClaim, instanceTypes, capacityType, options) if err != nil { return nil, err } @@ -139,74 +137,58 @@ func (p *DefaultProvider) EnsureAll(ctx context.Context, nodeClass *v1beta1.EC2N // InvalidateCache deletes a launch template from cache if it exists func (p *DefaultProvider) InvalidateCache(ctx context.Context, ltName string, ltID string) { - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", ltName, "launch-template-id", ltID)) + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("launch-template-name", ltName, "launch-template-id", ltID)) p.Lock() defer p.Unlock() defer p.cache.OnEvicted(p.cachedEvictedFunc(ctx)) p.cache.OnEvicted(nil) - logging.FromContext(ctx).Debugf("invalidating launch template in the cache because it no longer exists") + log.FromContext(ctx).V(1).Info("invalidating launch template in the cache because it no longer exists") p.cache.Delete(ltName) } func LaunchTemplateName(options *amifamily.LaunchTemplate) string { - return fmt.Sprintf("%s/%d", v1beta1.Group, lo.Must(hashstructure.Hash(options, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}))) + return fmt.Sprintf("%s/%d", apis.Group, lo.Must(hashstructure.Hash(options, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}))) } -func (p *DefaultProvider) createAMIOptions(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, labels, tags map[string]string) (*amifamily.Options, error) { +func (p *DefaultProvider) createAMIOptions(ctx context.Context, nodeClass *v1.EC2NodeClass, labels, tags map[string]string) (*amifamily.Options, error) { // Remove any labels passed into userData that are prefixed with "node-restriction.kubernetes.io" or "kops.k8s.io" since the kubelet can't // register the node with any labels from this domain: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#noderestriction for k := range labels { - labelDomain := corev1beta1.GetLabelDomain(k) - if strings.HasSuffix(labelDomain, v1.LabelNamespaceNodeRestriction) || strings.HasSuffix(labelDomain, "kops.k8s.io") { + labelDomain := karpv1.GetLabelDomain(k) + if strings.HasSuffix(labelDomain, corev1.LabelNamespaceNodeRestriction) || strings.HasSuffix(labelDomain, "kops.k8s.io") { delete(labels, k) } } - instanceProfile, err := p.getInstanceProfile(nodeClass) - if err != nil { - return nil, err - } + // Relying on the status rather than an API call means that Karpenter is subject to a race + // condition where EC2NodeClass spec changes haven't propagated to the status once a node + // has launched. + // If a user changes their EC2NodeClass and shortly after Karpenter launches a node, + // in the worst case, the node could be drifted and re-created. + // TODO @aengeda: add status generation fields to gate node creation until the status is updated from a spec change // Get constrained security groups - securityGroups, err := p.securityGroupProvider.List(ctx, nodeClass) - if err != nil { - return nil, err - } - if len(securityGroups) == 0 { - return nil, fmt.Errorf("no security groups exist given constraints") - } - options := &amifamily.Options{ - ClusterName: options.FromContext(ctx).ClusterName, - ClusterEndpoint: p.ClusterEndpoint, - ClusterCIDR: p.ClusterCIDR.Load(), - InstanceProfile: instanceProfile, - InstanceStorePolicy: nodeClass.Spec.InstanceStorePolicy, - SecurityGroups: lo.Map(securityGroups, func(s *ec2.SecurityGroup, _ int) v1beta1.SecurityGroup { - return v1beta1.SecurityGroup{ID: aws.StringValue(s.GroupId), Name: aws.StringValue(s.GroupName)} - }), - Tags: tags, - Labels: labels, - CABundle: p.CABundle, - KubeDNSIP: p.KubeDNSIP, - NodeClassName: nodeClass.Name, - } - if nodeClass.Spec.AssociatePublicIPAddress != nil { - options.AssociatePublicIPAddress = nodeClass.Spec.AssociatePublicIPAddress - } else if ok, err := p.subnetProvider.CheckAnyPublicIPAssociations(ctx, nodeClass); err != nil { - return nil, err - } else if !ok { - // when `AssociatePublicIPAddress` is not specified in the `EC2NodeClass` spec, - // If all referenced subnets do not assign public IPv4 addresses to EC2 instances therein, we explicitly set - // AssociatePublicIPAddress to 'false' in the Launch Template, generated based on this configuration struct. - // This is done to help comply with AWS account policies that require explicitly setting of that field to 'false'. - // https://github.com/aws/karpenter-provider-aws/issues/3815 - options.AssociatePublicIPAddress = aws.Bool(false) - } - return options, nil + if len(nodeClass.Status.SecurityGroups) == 0 { + return nil, fmt.Errorf("no security groups are present in the status") + } + return &amifamily.Options{ + ClusterName: options.FromContext(ctx).ClusterName, + ClusterEndpoint: p.ClusterEndpoint, + ClusterCIDR: p.ClusterCIDR.Load(), + InstanceProfile: nodeClass.Status.InstanceProfile, + InstanceStorePolicy: nodeClass.Spec.InstanceStorePolicy, + SecurityGroups: nodeClass.Status.SecurityGroups, + Tags: tags, + Labels: labels, + CABundle: p.CABundle, + KubeDNSIP: p.KubeDNSIP, + AssociatePublicIPAddress: nodeClass.Spec.AssociatePublicIPAddress, + NodeClassName: nodeClass.Name, + }, nil } func (p *DefaultProvider) ensureLaunchTemplate(ctx context.Context, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) { var launchTemplate *ec2.LaunchTemplate name := LaunchTemplateName(options) - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", name)) + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("launch-template-name", name)) // Read from cache if launchTemplate, ok := p.cache.Get(name); ok { p.cache.SetDefault(name, launchTemplate) @@ -228,7 +210,7 @@ func (p *DefaultProvider) ensureLaunchTemplate(ctx context.Context, options *ami return nil, fmt.Errorf("expected to find one launch template, but found %d", len(output.LaunchTemplates)) } else { if p.cm.HasChanged("launchtemplate-"+name, name) { - logging.FromContext(ctx).Debugf("discovered launch template") + log.FromContext(ctx).V(1).Info("discovered launch template") } launchTemplate = output.LaunchTemplates[0] } @@ -245,7 +227,7 @@ func (p *DefaultProvider) createLaunchTemplate(ctx context.Context, options *ami {ResourceType: aws.String(ec2.ResourceTypeNetworkInterface), Tags: utils.MergeTags(options.Tags)}, } // Add the spot-instances-request tag if trying to launch spot capacity - if options.CapacityType == corev1beta1.CapacityTypeSpot { + if options.CapacityType == karpv1.CapacityTypeSpot { launchTemplateDataTags = append(launchTemplateDataTags, &ec2.LaunchTemplateTagSpecificationRequest{ResourceType: aws.String(ec2.ResourceTypeSpotInstancesRequest), Tags: utils.MergeTags(options.Tags)}) } networkInterfaces := p.generateNetworkInterfaces(options) @@ -260,7 +242,7 @@ func (p *DefaultProvider) createLaunchTemplate(ctx context.Context, options *ami Enabled: aws.Bool(options.DetailedMonitoring), }, // If the network interface is defined, the security groups are defined within it - SecurityGroupIds: lo.Ternary(networkInterfaces != nil, nil, lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) })), + SecurityGroupIds: lo.Ternary(networkInterfaces != nil, nil, lo.Map(options.SecurityGroups, func(s v1.SecurityGroup, _ int) *string { return aws.String(s.ID) })), UserData: aws.String(userData), ImageId: aws.String(options.AMIID), MetadataOptions: &ec2.LaunchTemplateInstanceMetadataOptionsRequest{ @@ -275,14 +257,14 @@ func (p *DefaultProvider) createLaunchTemplate(ctx context.Context, options *ami TagSpecifications: []*ec2.TagSpecification{ { ResourceType: aws.String(ec2.ResourceTypeLaunchTemplate), - Tags: utils.MergeTags(options.Tags, map[string]string{karpenterManagedTagKey: options.ClusterName, v1beta1.LabelNodeClass: options.NodeClassName}), + Tags: utils.MergeTags(options.Tags, map[string]string{v1.TagManagedLaunchTemplate: options.ClusterName, v1.LabelNodeClass: options.NodeClassName}), }, }, }) if err != nil { return nil, err } - logging.FromContext(ctx).With("id", aws.StringValue(output.LaunchTemplate.LaunchTemplateId)).Debugf("created launch template") + log.FromContext(ctx).WithValues("id", aws.StringValue(output.LaunchTemplate.LaunchTemplateId)).V(1).Info("created launch template") return output.LaunchTemplate, nil } @@ -295,7 +277,7 @@ func (p *DefaultProvider) generateNetworkInterfaces(options *amifamily.LaunchTem // Some networking magic to ensure that one network card has higher priority than all the others (important if an instance needs a public IP w/o adding an EIP to every network card) DeviceIndex: lo.ToPtr(lo.Ternary[int64](i == 0, 0, 1)), InterfaceType: lo.ToPtr(ec2.NetworkInterfaceTypeEfa), - Groups: lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }), + Groups: lo.Map(options.SecurityGroups, func(s v1.SecurityGroup, _ int) *string { return aws.String(s.ID) }), // Instances launched with multiple pre-configured network interfaces cannot set AssociatePublicIPAddress to true. This is an EC2 limitation. However, this does not apply for instances // with a single EFA network interface, and we should support those use cases. Launch failures with multiple enis should be considered user misconfiguration. AssociatePublicIpAddress: options.AssociatePublicIPAddress, @@ -308,14 +290,14 @@ func (p *DefaultProvider) generateNetworkInterfaces(options *amifamily.LaunchTem { AssociatePublicIpAddress: options.AssociatePublicIPAddress, DeviceIndex: aws.Int64(0), - Groups: lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }), + Groups: lo.Map(options.SecurityGroups, func(s v1.SecurityGroup, _ int) *string { return aws.String(s.ID) }), }, } } return nil } -func (p *DefaultProvider) blockDeviceMappings(blockDeviceMappings []*v1beta1.BlockDeviceMapping) []*ec2.LaunchTemplateBlockDeviceMappingRequest { +func (p *DefaultProvider) blockDeviceMappings(blockDeviceMappings []*v1.BlockDeviceMapping) []*ec2.LaunchTemplateBlockDeviceMappingRequest { if len(blockDeviceMappings) == 0 { // The EC2 API fails with empty slices and expects nil. return nil @@ -352,18 +334,18 @@ func (p *DefaultProvider) volumeSize(quantity *resource.Quantity) *int64 { // Any error during hydration will result in a panic func (p *DefaultProvider) hydrateCache(ctx context.Context) { clusterName := options.FromContext(ctx).ClusterName - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("tag-key", karpenterManagedTagKey, "tag-value", clusterName)) + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("tag-key", v1.TagManagedLaunchTemplate, "tag-value", clusterName)) if err := p.ec2api.DescribeLaunchTemplatesPagesWithContext(ctx, &ec2.DescribeLaunchTemplatesInput{ - Filters: []*ec2.Filter{{Name: aws.String(fmt.Sprintf("tag:%s", karpenterManagedTagKey)), Values: []*string{aws.String(clusterName)}}}, + Filters: []*ec2.Filter{{Name: aws.String(fmt.Sprintf("tag:%s", v1.TagManagedLaunchTemplate)), Values: []*string{aws.String(clusterName)}}}, }, func(output *ec2.DescribeLaunchTemplatesOutput, _ bool) bool { for _, lt := range output.LaunchTemplates { p.cache.SetDefault(*lt.LaunchTemplateName, lt) } return true }); err != nil { - logging.FromContext(ctx).Errorf(fmt.Sprintf("Unable to hydrate the AWS launch template cache, %s", err)) + log.FromContext(ctx).Error(err, "unable to hydrate the AWS launch template cache") } else { - logging.FromContext(ctx).With("count", p.cache.ItemCount()).Debugf("hydrated launch template cache") + log.FromContext(ctx).WithValues("count", p.cache.ItemCount()).V(1).Info("hydrated launch template cache") } } @@ -376,36 +358,23 @@ func (p *DefaultProvider) cachedEvictedFunc(ctx context.Context) func(string, in } launchTemplate := lt.(*ec2.LaunchTemplate) if _, err := p.ec2api.DeleteLaunchTemplateWithContext(ctx, &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: launchTemplate.LaunchTemplateId}); awserrors.IgnoreNotFound(err) != nil { - logging.FromContext(ctx).With("launch-template", launchTemplate.LaunchTemplateName).Errorf("failed to delete launch template, %v", err) + log.FromContext(ctx).WithValues("launch-template", launchTemplate.LaunchTemplateName).Error(err, "failed to delete launch template") return } - logging.FromContext(ctx).With( + log.FromContext(ctx).WithValues( "id", aws.StringValue(launchTemplate.LaunchTemplateId), "name", aws.StringValue(launchTemplate.LaunchTemplateName), - ).Debugf("deleted launch template") - } -} - -func (p *DefaultProvider) getInstanceProfile(nodeClass *v1beta1.EC2NodeClass) (string, error) { - if nodeClass.Spec.InstanceProfile != nil { - return aws.StringValue(nodeClass.Spec.InstanceProfile), nil - } - if nodeClass.Spec.Role != "" { - if nodeClass.Status.InstanceProfile == "" { - return "", cloudprovider.NewNodeClassNotReadyError(fmt.Errorf("instance profile hasn't resolved for role")) - } - return nodeClass.Status.InstanceProfile, nil + ).V(1).Info("deleted launch template") } - return "", errors.New("neither spec.instanceProfile or spec.role is specified") } -func (p *DefaultProvider) DeleteAll(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) error { +func (p *DefaultProvider) DeleteAll(ctx context.Context, nodeClass *v1.EC2NodeClass) error { clusterName := options.FromContext(ctx).ClusterName var ltNames []*string if err := p.ec2api.DescribeLaunchTemplatesPagesWithContext(ctx, &ec2.DescribeLaunchTemplatesInput{ Filters: []*ec2.Filter{ - {Name: aws.String(fmt.Sprintf("tag:%s", karpenterManagedTagKey)), Values: []*string{aws.String(clusterName)}}, - {Name: aws.String(fmt.Sprintf("tag:%s", v1beta1.LabelNodeClass)), Values: []*string{aws.String(nodeClass.Name)}}, + {Name: aws.String(fmt.Sprintf("tag:%s", v1.TagManagedLaunchTemplate)), Values: []*string{aws.String(clusterName)}}, + {Name: aws.String(fmt.Sprintf("tag:%s", v1.LabelNodeClass)), Values: []*string{aws.String(nodeClass.Name)}}, }, }, func(output *ec2.DescribeLaunchTemplatesOutput, _ bool) bool { for _, lt := range output.LaunchTemplates { @@ -422,7 +391,7 @@ func (p *DefaultProvider) DeleteAll(ctx context.Context, nodeClass *v1beta1.EC2N deleteErr = multierr.Append(deleteErr, err) } if len(ltNames) > 0 { - logging.FromContext(ctx).With("launchTemplates", utils.PrettySlice(aws.StringValueSlice(ltNames), 5)).Debugf("deleted launch templates") + log.FromContext(ctx).WithValues("launchTemplates", utils.PrettySlice(aws.StringValueSlice(ltNames), 5)).V(1).Info("deleted launch templates") } if deleteErr != nil { return fmt.Errorf("deleting launch templates, %w", deleteErr) @@ -442,12 +411,12 @@ func (p *DefaultProvider) ResolveClusterCIDR(ctx context.Context) error { } if ipv4CIDR := out.Cluster.KubernetesNetworkConfig.ServiceIpv4Cidr; ipv4CIDR != nil { p.ClusterCIDR.Store(ipv4CIDR) - logging.FromContext(ctx).With("cluster-cidr", *ipv4CIDR).Debugf("discovered cluster CIDR") + log.FromContext(ctx).WithValues("cluster-cidr", *ipv4CIDR).V(1).Info("discovered cluster CIDR") return nil } if ipv6CIDR := out.Cluster.KubernetesNetworkConfig.ServiceIpv6Cidr; ipv6CIDR != nil { p.ClusterCIDR.Store(ipv6CIDR) - logging.FromContext(ctx).With("cluster-cidr", *ipv6CIDR).Debugf("discovered cluster CIDR") + log.FromContext(ctx).WithValues("cluster-cidr", *ipv6CIDR).V(1).Info("discovered cluster CIDR") return nil } return fmt.Errorf("no CIDR found in DescribeCluster response") diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 17449addc88f..c6ca8b6d9c55 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -26,37 +26,42 @@ import ( "testing" "time" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/ec2" admv1alpha1 "github.com/awslabs/amazon-eks-ami/nodeadm/api/v1alpha1" + "github.com/awslabs/operatorpkg/object" + opstatus "github.com/awslabs/operatorpkg/status" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" - . "knative.dev/pkg/logging/testing" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/controllers/provisioning" "sigs.k8s.io/karpenter/pkg/controllers/state" "sigs.k8s.io/karpenter/pkg/events" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" + . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" + "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" @@ -83,7 +88,7 @@ func TestAWS(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) ctx, stop = context.WithCancel(ctx) @@ -91,8 +96,8 @@ var _ = BeforeSuite(func() { fakeClock = &clock.FakeClock{} cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), - env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider, awsEnv.SubnetProvider) - cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) + env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider) + cluster = state.NewCluster(fakeClock, env.Client) prov = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) }) @@ -117,72 +122,147 @@ var _ = AfterEach(func() { }) var _ = Describe("LaunchTemplate Provider", func() { - var nodePool *corev1beta1.NodePool - var nodeClass *v1beta1.EC2NodeClass + var nodePool *karpv1.NodePool + var nodeClass *v1.EC2NodeClass BeforeEach(func() { - nodeClass = test.EC2NodeClass() - nodePool = coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - ObjectMeta: corev1beta1.ObjectMeta{ + nodeClass = test.EC2NodeClass( + v1.EC2NodeClass{ + Status: v1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + SecurityGroups: []v1.SecurityGroup{ + { + ID: "sg-test1", + }, + { + ID: "sg-test2", + }, + { + ID: "sg-test3", + }, + }, + Subnets: []v1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + }, + }, + }, + ) + nodeClass.StatusConditions().SetTrue(opstatus.ConditionReady) + nodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + ObjectMeta: karpv1.ObjectMeta{ // TODO @joinnis: Move this into the coretest.NodePool function Labels: map[string]string{coretest.DiscoveryLabel: "unspecified"}, }, - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Spec: karpv1.NodeClaimTemplateSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, }, }, }, - Kubelet: &corev1beta1.KubeletConfiguration{}, - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }, }, }) + _, err := awsEnv.SubnetProvider.List(ctx, nodeClass) // Hydrate the subnet cache + Expect(err).To(BeNil()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) + Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) }) It("should create unique launch templates for multiple identical nodeClasses", func() { - nodeClass2 := test.EC2NodeClass() - nodePool2 := coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodeClass2 := test.EC2NodeClass(v1.EC2NodeClass{ + Status: v1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + Subnets: nodeClass.Status.Subnets, + SecurityGroups: nodeClass.Status.SecurityGroups, + AMIs: nodeClass.Status.AMIs, + }, + }) + _, err := awsEnv.SubnetProvider.List(ctx, nodeClass2) // Hydrate the subnet cache + Expect(err).To(BeNil()) + nodePool2 := coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeSpot}, }, }, }, - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass2.Name, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass2).Group, + Kind: object.GVK(nodeClass2).Kind, + Name: nodeClass2.Name, }, }, }, }, }) - pods := []*v1.Pod{ - coretest.UnschedulablePod(coretest.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + nodeClass2.Status.SecurityGroups = []v1.SecurityGroup{ + { + ID: "sg-test1", + }, + { + ID: "sg-test2", + }, + { + ID: "sg-test3", + }, + } + nodeClass2.Status.Subnets = []v1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + } + nodeClass2.StatusConditions().SetTrue(opstatus.ConditionReady) + + pods := []*corev1.Pod{ + coretest.UnschedulablePod(coretest.PodOptions{NodeRequirements: []corev1.NodeSelectorRequirement{ { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeSpot}, }, }, }), - coretest.UnschedulablePod(coretest.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + coretest.UnschedulablePod(coretest.PodOptions{NodeRequirements: []corev1.NodeSelectorRequirement{ { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, }, }, }), @@ -194,7 +274,7 @@ var _ = Describe("LaunchTemplate Provider", func() { nodeClasses := [2]string{nodeClass.Name, nodeClass2.Name} awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.ForEach(func(ltInput *ec2.CreateLaunchTemplateInput) { for _, value := range ltInput.LaunchTemplateData.TagSpecifications[0].Tags { - if *value.Key == v1beta1.LabelNodeClass { + if *value.Key == v1.LabelNodeClass { Expect(*value.Value).To(BeElementOf(nodeClasses)) } } @@ -219,8 +299,9 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(*launchTemplate.LaunchTemplateSpecification.Version).To(Equal("$Latest")) }) }) - It("should fail to provision if the instance profile isn't defined", func() { + It("should fail to provision if the instance profile isn't ready", func() { nodeClass.Status.InstanceProfile = "" + nodeClass.StatusConditions().SetFalse(v1.ConditionTypeInstanceProfileReady, "reason", "message") ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -229,6 +310,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should use the instance profile on the EC2NodeClass when specified", func() { nodeClass.Spec.Role = "" nodeClass.Spec.InstanceProfile = aws.String("overridden-profile") + nodeClass.Status.InstanceProfile = "overridden-profile" ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -240,19 +322,19 @@ var _ = Describe("LaunchTemplate Provider", func() { }) Context("Cache", func() { It("should use same launch template for equivalent constraints", func() { - t1 := v1.Toleration{ + t1 := corev1.Toleration{ Key: "Abacus", Operator: "Equal", Value: "Zebra", Effect: "NoSchedule", } - t2 := v1.Toleration{ + t2 := corev1.Toleration{ Key: "Zebra", Operator: "Equal", Value: "Abacus", Effect: "NoSchedule", } - t3 := v1.Toleration{ + t3 := corev1.Toleration{ Key: "Boar", Operator: "Equal", Value: "Abacus", @@ -260,17 +342,17 @@ var _ = Describe("LaunchTemplate Provider", func() { } // constrain the packer to a single launch template type - rr := v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("24"), - v1beta1.ResourceNVIDIAGPU: resource.MustParse("1"), + rr := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("24"), + v1.ResourceNVIDIAGPU: resource.MustParse("1"), }, - Limits: v1.ResourceList{v1beta1.ResourceNVIDIAGPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{v1.ResourceNVIDIAGPU: resource.MustParse("1")}, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod1 := coretest.UnschedulablePod(coretest.PodOptions{ - Tolerations: []v1.Toleration{t1, t2, t3}, + Tolerations: []corev1.Toleration{t1, t2, t3}, ResourceRequirements: rr, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod1) @@ -283,7 +365,7 @@ var _ = Describe("LaunchTemplate Provider", func() { } pod2 := coretest.UnschedulablePod(coretest.PodOptions{ - Tolerations: []v1.Toleration{t2, t3, t1}, + Tolerations: []corev1.Toleration{t2, t3, t1}, ResourceRequirements: rr, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod2) @@ -298,7 +380,7 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(lts1.Equal(lts2)).To(BeTrue()) }) It("should recover from an out-of-sync launch template cache", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: aws.Int32(1)} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{MaxPods: aws.Int32(1)} ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -328,8 +410,8 @@ var _ = Describe("LaunchTemplate Provider", func() { {ClusterEndpoint: "test-endpoint"}, {ClusterCIDR: lo.ToPtr("test-cidr")}, {InstanceProfile: "test-profile"}, - {InstanceStorePolicy: lo.ToPtr(v1beta1.InstanceStorePolicyRAID0)}, - {SecurityGroups: []v1beta1.SecurityGroup{{Name: "test-sg"}}}, + {InstanceStorePolicy: lo.ToPtr(v1.InstanceStorePolicyRAID0)}, + {SecurityGroups: []v1.SecurityGroup{{Name: "test-sg"}}}, {Tags: map[string]string{"test-key": "test-value"}}, {KubeDNSIP: net.ParseIP("192.0.0.2")}, {AssociatePublicIPAddress: lo.ToPtr(true)}, @@ -358,10 +440,10 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(lo.Uniq(launchtemplateResult)[0]).To(Equal(launchtemplate.LaunchTemplateName(&amifamily.LaunchTemplate{Options: &amifamily.Options{}}))) }) It("should generate different launch template names based on kubelet configuration", func() { - kubeletChanges := []*corev1beta1.KubeletConfiguration{ + kubeletChanges := []*v1.KubeletConfiguration{ {}, - {KubeReserved: map[string]string{string(v1.ResourceCPU): "20"}}, - {SystemReserved: map[string]string{string(v1.ResourceMemory): "10Gi"}}, + {KubeReserved: map[string]string{string(corev1.ResourceCPU): "20"}}, + {SystemReserved: map[string]string{string(corev1.ResourceMemory): "10Gi"}}, {EvictionHard: map[string]string{"memory.available": "52%"}}, {EvictionSoft: map[string]string{"nodefs.available": "132%"}}, {MaxPods: aws.Int32(20)}, @@ -380,7 +462,7 @@ var _ = Describe("LaunchTemplate Provider", func() { {ClusterName: "test-name"}, {ClusterEndpoint: "test-endpoint"}, {ClusterCIDR: lo.ToPtr("test-cidr")}, - {Taints: []v1.Taint{{Key: "test-key", Value: "test-value"}}}, + {Taints: []corev1.Taint{{Key: "test-key", Value: "test-value"}}}, {Labels: map[string]string{"test-key": "test-value"}}, {CABundle: lo.ToPtr("test-bundle")}, {AWSENILimitedPodDensity: true}, @@ -398,7 +480,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should generate different launch template names based on launchtemplate option configuration", func() { launchtemplates := []*amifamily.LaunchTemplate{ {}, - {BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: lo.ToPtr("test-block")}}}, + {BlockDeviceMappings: []*v1.BlockDeviceMapping{{DeviceName: lo.ToPtr("test-block")}}}, {AMIID: "test-ami"}, {DetailedMonitoring: true}, {EFACount: 12}, @@ -430,9 +512,9 @@ var _ = Describe("LaunchTemplate Provider", func() { pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKey(v1.LabelOSStable)) - Expect(node.Labels).To(HaveKey(v1.LabelArchStable)) - Expect(node.Labels).To(HaveKey(v1.LabelInstanceTypeStable)) + Expect(node.Labels).To(HaveKey(corev1.LabelOSStable)) + Expect(node.Labels).To(HaveKey(corev1.LabelArchStable)) + Expect(node.Labels).To(HaveKey(corev1.LabelInstanceTypeStable)) }) }) Context("Tags", func() { @@ -464,12 +546,12 @@ var _ = Describe("LaunchTemplate Provider", func() { "tag1": "tag1value", "tag2": "tag2value", } - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeSpot}, }, }, } @@ -514,7 +596,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) Context("Block Device Mappings", func() { It("should default AL2 block device mappings", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -528,7 +610,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should default AL2023 block device mappings", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2023@latest"}} awsEnv.LaunchTemplateProvider.CABundle = lo.ToPtr("Y2EtYnVuZGxlCg==") awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(lo.ToPtr("10.100.0.0/16")) ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -544,11 +626,11 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should use custom block device mapping", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 - nodeClass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} + nodeClass.Spec.BlockDeviceMappings = []*v1.BlockDeviceMapping{ { DeviceName: aws.String("/dev/xvda"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ DeleteOnTermination: aws.Bool(true), Encrypted: aws.Bool(true), VolumeType: aws.String("io2"), @@ -559,7 +641,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, { DeviceName: aws.String("/dev/xvdb"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ DeleteOnTermination: aws.Bool(true), Encrypted: aws.Bool(true), VolumeType: aws.String("io2"), @@ -594,11 +676,11 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should round up for custom block device mappings when specified in gigabytes", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 - nodeClass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} + nodeClass.Spec.BlockDeviceMappings = []*v1.BlockDeviceMapping{ { DeviceName: aws.String("/dev/xvda"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ DeleteOnTermination: aws.Bool(true), Encrypted: aws.Bool(true), VolumeType: aws.String("io2"), @@ -609,7 +691,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, { DeviceName: aws.String("/dev/xvdb"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ DeleteOnTermination: aws.Bool(true), Encrypted: aws.Bool(true), VolumeType: aws.String("io2"), @@ -631,7 +713,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should default bottlerocket second volume with root volume size", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -650,8 +732,8 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should not default block device mappings for custom AMIFamilies", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -662,12 +744,12 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should use custom block device mapping for custom AMIFamilies", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} - nodeClass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Spec.BlockDeviceMappings = []*v1.BlockDeviceMapping{ { DeviceName: aws.String("/dev/xvda"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ DeleteOnTermination: aws.Bool(true), Encrypted: aws.Bool(true), VolumeType: aws.String("io2"), @@ -697,10 +779,10 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should pack pods when a daemonset has an ephemeral-storage request", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass, coretest.DaemonSet( coretest.DaemonSetOptions{PodOptions: coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi"), - v1.ResourceEphemeralStorage: resource.MustParse("1Gi")}}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi")}}, }}, )) pod := coretest.UnschedulablePod() @@ -709,57 +791,57 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should pack pods with any ephemeral-storage request", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceEphemeralStorage: resource.MustParse("1G"), + pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceEphemeralStorage: resource.MustParse("1G"), }}}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) }) It("should pack pods with large ephemeral-storage request", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceEphemeralStorage: resource.MustParse("10Gi"), + pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceEphemeralStorage: resource.MustParse("10Gi"), }}}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) }) It("should not pack pods if the sum of pod ephemeral-storage and overhead exceeds node capacity", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceEphemeralStorage: resource.MustParse("19Gi"), + pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceEphemeralStorage: resource.MustParse("19Gi"), }}}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) }) It("should pack pods if the pod's ephemeral-storage exceeds node capacity and instance storage is mounted", func() { - nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1beta1.InstanceStorePolicyRAID0) + nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1.InstanceStorePolicyRAID0) ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ + pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ // Default node ephemeral-storage capacity is 20Gi - v1.ResourceEphemeralStorage: resource.MustParse("5000Gi"), + corev1.ResourceEphemeralStorage: resource.MustParse("5000Gi"), }}}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("m6idn.32xlarge")) + Expect(node.Labels[corev1.LabelInstanceTypeStable]).To(Equal("m6idn.32xlarge")) Expect(*node.Status.Capacity.StorageEphemeral()).To(Equal(resource.MustParse("7600G"))) }) It("should launch multiple nodes if sum of pod ephemeral-storage requests exceeds a single nodes capacity", func() { - var nodes []*v1.Node + var nodes []*corev1.Node ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pods := []*v1.Pod{ - coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceEphemeralStorage: resource.MustParse("10Gi"), + pods := []*corev1.Pod{ + coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceEphemeralStorage: resource.MustParse("10Gi"), }, }, }), - coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceEphemeralStorage: resource.MustParse("10Gi"), + coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceEphemeralStorage: resource.MustParse("10Gi"), }, }, }), @@ -772,16 +854,16 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should only pack pods with ephemeral-storage requests that will fit on an available node", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pods := []*v1.Pod{ - coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceEphemeralStorage: resource.MustParse("10Gi"), + pods := []*corev1.Pod{ + coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceEphemeralStorage: resource.MustParse("10Gi"), }, }, }), - coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceEphemeralStorage: resource.MustParse("150Gi"), + coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceEphemeralStorage: resource.MustParse("150Gi"), }, }, }), @@ -792,9 +874,9 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should not pack pod if no available instance types have enough storage", func() { ExpectApplied(ctx, env.Client, nodePool) - pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceEphemeralStorage: resource.MustParse("150Gi"), + pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceEphemeralStorage: resource.MustParse("150Gi"), }, }, }) @@ -802,24 +884,24 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectNotScheduled(ctx, env.Client, pod) }) It("should pack pods using the blockdevicemappings from the provider spec when defined", func() { - nodeClass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ + nodeClass.Spec.BlockDeviceMappings = []*v1.BlockDeviceMapping{ { DeviceName: aws.String("/dev/xvda"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ VolumeSize: resource.NewScaledQuantity(50, resource.Giga), }, }, { DeviceName: aws.String("/dev/xvdb"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ VolumeSize: resource.NewScaledQuantity(20, resource.Giga), }, }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceEphemeralStorage: resource.MustParse("25Gi"), + pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceEphemeralStorage: resource.MustParse("25Gi"), }, }, }) @@ -829,27 +911,27 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectScheduled(ctx, env.Client, pod) }) It("should pack pods using blockdevicemappings for Custom AMIFamily", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} - nodeClass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Spec.BlockDeviceMappings = []*v1.BlockDeviceMapping{ { DeviceName: aws.String("/dev/xvda"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ VolumeSize: resource.NewScaledQuantity(20, resource.Giga), }, }, { DeviceName: aws.String("/dev/xvdb"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ VolumeSize: resource.NewScaledQuantity(40, resource.Giga), }, }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ + pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ // this pod can only be satisfied if `/dev/xvdb` will house all the pods. - v1.ResourceEphemeralStorage: resource.MustParse("25Gi"), + corev1.ResourceEphemeralStorage: resource.MustParse("25Gi"), }, }, }) @@ -859,34 +941,34 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectScheduled(ctx, env.Client, pod) }) It("should pack pods using the configured root volume in blockdevicemappings", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} - nodeClass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Spec.BlockDeviceMappings = []*v1.BlockDeviceMapping{ { DeviceName: aws.String("/dev/xvda"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ VolumeSize: resource.NewScaledQuantity(20, resource.Giga), }, }, { DeviceName: aws.String("/dev/xvdb"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ VolumeSize: resource.NewScaledQuantity(40, resource.Giga), }, RootVolume: true, }, { DeviceName: aws.String("/dev/xvdc"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ VolumeSize: resource.NewScaledQuantity(20, resource.Giga), }, }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ + pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ // this pod can only be satisfied if `/dev/xvdb` will house all the pods. - v1.ResourceEphemeralStorage: resource.MustParse("25Gi"), + corev1.ResourceEphemeralStorage: resource.MustParse("25Gi"), }, }, }) @@ -928,19 +1010,20 @@ var _ = Describe("LaunchTemplate Provider", func() { VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, "", nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -981,19 +1064,20 @@ var _ = Describe("LaunchTemplate Provider", func() { VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, "", nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -1006,20 +1090,20 @@ var _ = Describe("LaunchTemplate Provider", func() { VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} + amiFamily := amifamily.GetAMIFamily(nodeClass.AMIFamily(), &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, "", nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - nodePool.Spec.Template.Spec.Kubelet.MaxPods, - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, - nodePool.Spec.Template.Spec.Kubelet.KubeReserved, - nodePool.Spec.Template.Spec.Kubelet.SystemReserved, - nodePool.Spec.Template.Spec.Kubelet.EvictionHard, - nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + nodeClass.Spec.Kubelet.MaxPods, + nodeClass.Spec.Kubelet.PodsPerCore, + nodeClass.Spec.Kubelet.KubeReserved, + nodeClass.Spec.Kubelet.SystemReserved, + nodeClass.Spec.Kubelet.EvictionHard, + nodeClass.Spec.Kubelet.EvictionSoft, amiFamily, nil, ) @@ -1036,7 +1120,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectLaunchTemplatesCreatedWithUserDataContaining("--use-max-pods false") }) It("should specify --use-max-pods=false and --max-pods user value when user specifies maxPods in NodePool", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: aws.Int32(10)} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{MaxPods: aws.Int32(10)} ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1044,11 +1128,11 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectLaunchTemplatesCreatedWithUserDataContaining("--use-max-pods false", "--max-pods=10") }) It("should specify --system-reserved when overriding system reserved values", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceCPU): "500m", - string(v1.ResourceMemory): "1Gi", - string(v1.ResourceEphemeralStorage): "2Gi", + string(corev1.ResourceCPU): "500m", + string(corev1.ResourceMemory): "1Gi", + string(corev1.ResourceEphemeralStorage): "2Gi", }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1065,17 +1149,17 @@ var _ = Describe("LaunchTemplate Provider", func() { i := strings.Index(string(userData), arg) rem := string(userData)[(i + len(arg)):] i = strings.Index(rem, "'") - for k, v := range nodePool.Spec.Template.Spec.Kubelet.SystemReserved { + for k, v := range nodeClass.Spec.Kubelet.SystemReserved { Expect(rem[:i]).To(ContainSubstring(fmt.Sprintf("%v=%v", k, v))) } }) }) It("should specify --kube-reserved when overriding system reserved values", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ KubeReserved: map[string]string{ - string(v1.ResourceCPU): "500m", - string(v1.ResourceMemory): "1Gi", - string(v1.ResourceEphemeralStorage): "2Gi", + string(corev1.ResourceCPU): "500m", + string(corev1.ResourceMemory): "1Gi", + string(corev1.ResourceEphemeralStorage): "2Gi", }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1092,13 +1176,13 @@ var _ = Describe("LaunchTemplate Provider", func() { i := strings.Index(string(userData), arg) rem := string(userData)[(i + len(arg)):] i = strings.Index(rem, "'") - for k, v := range nodePool.Spec.Template.Spec.Kubelet.KubeReserved { + for k, v := range nodeClass.Spec.Kubelet.KubeReserved { Expect(rem[:i]).To(ContainSubstring(fmt.Sprintf("%v=%v", k, v))) } }) }) It("should pass eviction hard threshold values when specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ EvictionHard: map[string]string{ "memory.available": "10%", "nodefs.available": "15%", @@ -1119,13 +1203,13 @@ var _ = Describe("LaunchTemplate Provider", func() { i := strings.Index(string(userData), arg) rem := string(userData)[(i + len(arg)):] i = strings.Index(rem, "'") - for k, v := range nodePool.Spec.Template.Spec.Kubelet.EvictionHard { + for k, v := range nodeClass.Spec.Kubelet.EvictionHard { Expect(rem[:i]).To(ContainSubstring(fmt.Sprintf("%v<%v", k, v))) } }) }) It("should pass eviction soft threshold values when specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ EvictionSoft: map[string]string{ "memory.available": "10%", "nodefs.available": "15%", @@ -1151,13 +1235,13 @@ var _ = Describe("LaunchTemplate Provider", func() { i := strings.Index(string(userData), arg) rem := string(userData)[(i + len(arg)):] i = strings.Index(rem, "'") - for k, v := range nodePool.Spec.Template.Spec.Kubelet.EvictionSoft { + for k, v := range nodeClass.Spec.Kubelet.EvictionSoft { Expect(rem[:i]).To(ContainSubstring(fmt.Sprintf("%v<%v", k, v))) } }) }) It("should pass eviction soft grace period values when specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ EvictionSoftGracePeriod: map[string]metav1.Duration{ "memory.available": {Duration: time.Minute}, "nodefs.available": {Duration: time.Second * 180}, @@ -1183,13 +1267,13 @@ var _ = Describe("LaunchTemplate Provider", func() { i := strings.Index(string(userData), arg) rem := string(userData)[(i + len(arg)):] i = strings.Index(rem, "'") - for k, v := range nodePool.Spec.Template.Spec.Kubelet.EvictionSoftGracePeriod { + for k, v := range nodeClass.Spec.Kubelet.EvictionSoftGracePeriod { Expect(rem[:i]).To(ContainSubstring(fmt.Sprintf("%v=%v", k, v.Duration.String()))) } }) }) It("should pass eviction max pod grace period when specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ EvictionMaxPodGracePeriod: aws.Int32(300), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1199,7 +1283,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectLaunchTemplatesCreatedWithUserDataContaining(fmt.Sprintf("--eviction-max-pod-grace-period=%d", 300)) }) It("should specify --pods-per-core", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ PodsPerCore: aws.Int32(2), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1209,7 +1293,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectLaunchTemplatesCreatedWithUserDataContaining(fmt.Sprintf("--pods-per-core=%d", 2)) }) It("should specify --pods-per-core with --max-pods enabled", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ PodsPerCore: aws.Int32(2), MaxPods: aws.Int32(100), } @@ -1236,7 +1320,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectLaunchTemplatesCreatedWithUserDataContaining("--dns-cluster-ip '10.0.100.10'") }) It("should pass ImageGCHighThresholdPercent when specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ ImageGCHighThresholdPercent: aws.Int32(50), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1246,7 +1330,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectLaunchTemplatesCreatedWithUserDataContaining("--image-gc-high-threshold=50") }) It("should pass ImageGCLowThresholdPercent when specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ ImageGCLowThresholdPercent: aws.Int32(50), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1256,7 +1340,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectLaunchTemplatesCreatedWithUserDataContaining("--image-gc-low-threshold=50") }) It("should pass --cpu-fs-quota when specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ CPUCFSQuota: aws.Bool(false), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1267,19 +1351,19 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should not pass any labels prefixed with the node-restriction.kubernetes.io domain", func() { nodePool.Spec.Template.Labels = lo.Assign(nodePool.Spec.Template.Labels, map[string]string{ - v1.LabelNamespaceNodeRestriction + "/team": "team-1", - v1.LabelNamespaceNodeRestriction + "/custom-label": "custom-value", - "subdomain." + v1.LabelNamespaceNodeRestriction + "/custom-label": "custom-value", + corev1.LabelNamespaceNodeRestriction + "/team": "team-1", + corev1.LabelNamespaceNodeRestriction + "/custom-label": "custom-value", + "subdomain." + corev1.LabelNamespaceNodeRestriction + "/custom-label": "custom-value", }) ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) - ExpectLaunchTemplatesCreatedWithUserDataNotContaining(v1.LabelNamespaceNodeRestriction) + ExpectLaunchTemplatesCreatedWithUserDataNotContaining(corev1.LabelNamespaceNodeRestriction) }) It("should specify --local-disks raid0 when instance-store policy is set on AL2", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 - nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1beta1.InstanceStorePolicyRAID0) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} + nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1.InstanceStorePolicyRAID0) ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1288,41 +1372,45 @@ var _ = Describe("LaunchTemplate Provider", func() { }) Context("Bottlerocket", func() { BeforeEach(func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} }) It("should merge in custom user data", func() { content, err := os.ReadFile("testdata/br_userdata_input.golden") Expect(err).To(BeNil()) - nodeClass.Spec.UserData = aws.String(fmt.Sprintf(string(content), corev1beta1.NodePoolLabelKey)) - nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectNoExecute}} - nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "baz", Value: "bin", Effect: v1.TaintEffectNoExecute}} + nodeClass.Spec.UserData = aws.String(fmt.Sprintf(string(content), karpv1.NodePoolLabelKey)) + nodePool.Spec.Template.Spec.Taints = []corev1.Taint{{Key: "foo", Value: "bar", Effect: corev1.TaintEffectNoExecute}} + nodePool.Spec.Template.Spec.StartupTaints = []corev1.Taint{{Key: "baz", Value: "bin", Effect: corev1.TaintEffectNoExecute}} ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod(coretest.PodOptions{ - Tolerations: []v1.Toleration{{Operator: v1.TolerationOpExists}}, + Tolerations: []corev1.Toleration{{Operator: corev1.TolerationOpExists}}, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) content, err = os.ReadFile("testdata/br_userdata_merged.golden") Expect(err).To(BeNil()) - ExpectLaunchTemplatesCreatedWithUserData(fmt.Sprintf(string(content), corev1beta1.NodePoolLabelKey, nodePool.Name)) + ExpectLaunchTemplatesCreatedWithUserData(fmt.Sprintf(string(content), karpv1.NodePoolLabelKey, nodePool.Name)) }) It("should bootstrap when custom user data is empty", func() { - nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectNoExecute}} - nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "baz", Value: "bin", Effect: v1.TaintEffectNoExecute}} + nodePool.Spec.Template.Spec.Taints = []corev1.Taint{{Key: "foo", Value: "bar", Effect: corev1.TaintEffectNoExecute}} + nodePool.Spec.Template.Spec.StartupTaints = []corev1.Taint{{Key: "baz", Value: "bin", Effect: corev1.TaintEffectNoExecute}} ExpectApplied(ctx, env.Client, nodeClass, nodePool) Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(nodePool), nodePool)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ - Tolerations: []v1.Toleration{{Operator: v1.TolerationOpExists}}, + Tolerations: []corev1.Toleration{{Operator: corev1.TolerationOpExists}}, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) content, err := os.ReadFile("testdata/br_userdata_unmerged.golden") Expect(err).To(BeNil()) - ExpectLaunchTemplatesCreatedWithUserData(fmt.Sprintf(string(content), corev1beta1.NodePoolLabelKey, nodePool.Name)) + ExpectLaunchTemplatesCreatedWithUserData(fmt.Sprintf(string(content), karpv1.NodePoolLabelKey, nodePool.Name)) }) It("should not bootstrap when provider ref points to a non-existent EC2NodeClass resource", func() { - nodePool.Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{Name: "doesnotexist"} + nodePool.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{ + Group: "doesnotexist", + Kind: "doesnotexist", + Name: "doesnotexist", + } ExpectApplied(ctx, env.Client, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1338,15 +1426,14 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectNotScheduled(ctx, env.Client, pod) }) It("should override system reserved values in user data", func() { - ExpectApplied(ctx, env.Client, nodeClass) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceCPU): "2", - string(v1.ResourceMemory): "3Gi", - string(v1.ResourceEphemeralStorage): "10Gi", + string(corev1.ResourceCPU): "2", + string(corev1.ResourceMemory): "3Gi", + string(corev1.ResourceEphemeralStorage): "10Gi", }, } - ExpectApplied(ctx, env.Client, nodePool) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1357,21 +1444,20 @@ var _ = Describe("LaunchTemplate Provider", func() { config := &bootstrap.BottlerocketConfig{} Expect(config.UnmarshalTOML(userData)).To(Succeed()) Expect(len(config.Settings.Kubernetes.SystemReserved)).To(Equal(3)) - Expect(config.Settings.Kubernetes.SystemReserved[v1.ResourceCPU.String()]).To(Equal("2")) - Expect(config.Settings.Kubernetes.SystemReserved[v1.ResourceMemory.String()]).To(Equal("3Gi")) - Expect(config.Settings.Kubernetes.SystemReserved[v1.ResourceEphemeralStorage.String()]).To(Equal("10Gi")) + Expect(config.Settings.Kubernetes.SystemReserved[corev1.ResourceCPU.String()]).To(Equal("2")) + Expect(config.Settings.Kubernetes.SystemReserved[corev1.ResourceMemory.String()]).To(Equal("3Gi")) + Expect(config.Settings.Kubernetes.SystemReserved[corev1.ResourceEphemeralStorage.String()]).To(Equal("10Gi")) }) }) It("should override kube reserved values in user data", func() { - ExpectApplied(ctx, env.Client, nodeClass) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ KubeReserved: map[string]string{ - string(v1.ResourceCPU): "2", - string(v1.ResourceMemory): "3Gi", - string(v1.ResourceEphemeralStorage): "10Gi", + string(corev1.ResourceCPU): "2", + string(corev1.ResourceMemory): "3Gi", + string(corev1.ResourceEphemeralStorage): "10Gi", }, } - ExpectApplied(ctx, env.Client, nodePool) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1382,21 +1468,20 @@ var _ = Describe("LaunchTemplate Provider", func() { config := &bootstrap.BottlerocketConfig{} Expect(config.UnmarshalTOML(userData)).To(Succeed()) Expect(len(config.Settings.Kubernetes.KubeReserved)).To(Equal(3)) - Expect(config.Settings.Kubernetes.KubeReserved[v1.ResourceCPU.String()]).To(Equal("2")) - Expect(config.Settings.Kubernetes.KubeReserved[v1.ResourceMemory.String()]).To(Equal("3Gi")) - Expect(config.Settings.Kubernetes.KubeReserved[v1.ResourceEphemeralStorage.String()]).To(Equal("10Gi")) + Expect(config.Settings.Kubernetes.KubeReserved[corev1.ResourceCPU.String()]).To(Equal("2")) + Expect(config.Settings.Kubernetes.KubeReserved[corev1.ResourceMemory.String()]).To(Equal("3Gi")) + Expect(config.Settings.Kubernetes.KubeReserved[corev1.ResourceEphemeralStorage.String()]).To(Equal("10Gi")) }) }) It("should override kube reserved values in user data", func() { - ExpectApplied(ctx, env.Client, nodeClass) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ EvictionHard: map[string]string{ "memory.available": "10%", "nodefs.available": "15%", "nodefs.inodesFree": "5%", }, } - ExpectApplied(ctx, env.Client, nodePool) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1413,7 +1498,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should specify max pods value when passing maxPods in configuration", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ MaxPods: aws.Int32(10), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1431,7 +1516,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should pass ImageGCHighThresholdPercent when specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ ImageGCHighThresholdPercent: aws.Int32(50), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1451,7 +1536,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should pass ImageGCLowThresholdPercent when specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ ImageGCLowThresholdPercent: aws.Int32(50), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1486,7 +1571,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should pass CPUCFSQuota when specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ CPUCFSQuota: aws.Bool(false), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1506,7 +1591,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) Context("AL2 Custom UserData", func() { BeforeEach(func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} }) It("should merge in custom user data", func() { content, err := os.ReadFile("testdata/al2_userdata_input.golden") @@ -1518,7 +1603,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectScheduled(ctx, env.Client, pod) content, err = os.ReadFile("testdata/al2_userdata_merged.golden") Expect(err).To(BeNil()) - expectedUserData := fmt.Sprintf(string(content), corev1beta1.NodePoolLabelKey, nodePool.Name) + expectedUserData := fmt.Sprintf(string(content), karpv1.NodePoolLabelKey, nodePool.Name) ExpectLaunchTemplatesCreatedWithUserData(expectedUserData) }) It("should merge in custom user data when Content-Type is before MIME-Version", func() { @@ -1531,7 +1616,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectScheduled(ctx, env.Client, pod) content, err = os.ReadFile("testdata/al2_userdata_merged.golden") Expect(err).To(BeNil()) - expectedUserData := fmt.Sprintf(string(content), corev1beta1.NodePoolLabelKey, nodePool.Name) + expectedUserData := fmt.Sprintf(string(content), karpv1.NodePoolLabelKey, nodePool.Name) ExpectLaunchTemplatesCreatedWithUserData(expectedUserData) }) It("should merge in custom user data not in multi-part mime format", func() { @@ -1544,7 +1629,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectScheduled(ctx, env.Client, pod) content, err = os.ReadFile("testdata/al2_userdata_merged.golden") Expect(err).To(BeNil()) - expectedUserData := fmt.Sprintf(string(content), corev1beta1.NodePoolLabelKey, nodePool.Name) + expectedUserData := fmt.Sprintf(string(content), karpv1.NodePoolLabelKey, nodePool.Name) ExpectLaunchTemplatesCreatedWithUserData(expectedUserData) }) It("should handle empty custom user data", func() { @@ -1555,13 +1640,13 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectScheduled(ctx, env.Client, pod) content, err := os.ReadFile("testdata/al2_userdata_unmerged.golden") Expect(err).To(BeNil()) - expectedUserData := fmt.Sprintf(string(content), corev1beta1.NodePoolLabelKey, nodePool.Name) + expectedUserData := fmt.Sprintf(string(content), karpv1.NodePoolLabelKey, nodePool.Name) ExpectLaunchTemplatesCreatedWithUserData(expectedUserData) }) }) Context("AL2023", func() { BeforeEach(func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2023@latest"}} // base64 encoded version of "ca-bundle" to ensure the nodeadm bootstrap provider can decode successfully awsEnv.LaunchTemplateProvider.CABundle = lo.ToPtr("Y2EtYnVuZGxlCg==") @@ -1569,21 +1654,21 @@ var _ = Describe("LaunchTemplate Provider", func() { }) Context("Kubelet", func() { It("should specify taints in the KubeletConfiguration when specified in NodePool", func() { - desiredTaints := []v1.Taint{ + desiredTaints := []corev1.Taint{ { Key: "test-taint-1", - Effect: v1.TaintEffectNoSchedule, + Effect: corev1.TaintEffectNoSchedule, }, { Key: "test-taint-2", - Effect: v1.TaintEffectNoExecute, + Effect: corev1.TaintEffectNoExecute, }, } nodePool.Spec.Template.Spec.Taints = desiredTaints ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.UnscheduleablePodOptions(coretest.PodOptions{ - Tolerations: []v1.Toleration{{ - Operator: v1.TolerationOpExists, + Tolerations: []corev1.Toleration{{ + Operator: corev1.TolerationOpExists, }}, })) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1593,10 +1678,10 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(len(configs)).To(Equal(1)) taintsRaw, ok := configs[0].Spec.Kubelet.Config["registerWithTaints"] Expect(ok).To(BeTrue()) - taints := []v1.Taint{} + taints := []corev1.Taint{} Expect(yaml.Unmarshal(taintsRaw.Raw, &taints)).To(Succeed()) - Expect(len(taints)).To(Equal(2)) - Expect(taints).To(ContainElements(lo.Map(desiredTaints, func(t v1.Taint, _ int) interface{} { + Expect(len(taints)).To(Equal(3)) + Expect(taints).To(ContainElements(lo.Map(desiredTaints, func(t corev1.Taint, _ int) interface{} { return interface{}(t) }))) } @@ -1626,8 +1711,8 @@ var _ = Describe("LaunchTemplate Provider", func() { }) DescribeTable( "should specify KubletConfiguration field when specified in NodePool", - func(field string, kc corev1beta1.KubeletConfiguration) { - nodePool.Spec.Template.Spec.Kubelet = &kc + func(field string, kc v1.KubeletConfiguration) { + nodeClass.Spec.Kubelet = &kc ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1651,28 +1736,28 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(configs[0].Spec.Kubelet.Config[field]).To(Equal(inlineConfig[field])) } }, - Entry("systemReserved", "systemReserved", corev1beta1.KubeletConfiguration{ + Entry("systemReserved", "systemReserved", v1.KubeletConfiguration{ SystemReserved: map[string]string{ - string(v1.ResourceCPU): "500m", - string(v1.ResourceMemory): "1Gi", - string(v1.ResourceEphemeralStorage): "2Gi", + string(corev1.ResourceCPU): "500m", + string(corev1.ResourceMemory): "1Gi", + string(corev1.ResourceEphemeralStorage): "2Gi", }, }), - Entry("kubeReserved", "kubeReserved", corev1beta1.KubeletConfiguration{ + Entry("kubeReserved", "kubeReserved", v1.KubeletConfiguration{ KubeReserved: map[string]string{ - string(v1.ResourceCPU): "500m", - string(v1.ResourceMemory): "1Gi", - string(v1.ResourceEphemeralStorage): "2Gi", + string(corev1.ResourceCPU): "500m", + string(corev1.ResourceMemory): "1Gi", + string(corev1.ResourceEphemeralStorage): "2Gi", }, }), - Entry("evictionHard", "evictionHard", corev1beta1.KubeletConfiguration{ + Entry("evictionHard", "evictionHard", v1.KubeletConfiguration{ EvictionHard: map[string]string{ "memory.available": "10%", "nodefs.available": "15%", "nodefs.inodesFree": "5%", }, }), - Entry("evictionSoft", "evictionSoft", corev1beta1.KubeletConfiguration{ + Entry("evictionSoft", "evictionSoft", v1.KubeletConfiguration{ EvictionSoft: map[string]string{ "memory.available": "10%", "nodefs.available": "15%", @@ -1684,7 +1769,7 @@ var _ = Describe("LaunchTemplate Provider", func() { "nodefs.inodesFree": {Duration: time.Minute * 5}, }, }), - Entry("evictionSoftGracePeriod", "evictionSoftGracePeriod", corev1beta1.KubeletConfiguration{ + Entry("evictionSoftGracePeriod", "evictionSoftGracePeriod", v1.KubeletConfiguration{ EvictionSoft: map[string]string{ "memory.available": "10%", "nodefs.available": "15%", @@ -1696,28 +1781,28 @@ var _ = Describe("LaunchTemplate Provider", func() { "nodefs.inodesFree": {Duration: time.Minute * 5}, }, }), - Entry("evictionMaxPodGracePeriod", "evictionMaxPodGracePeriod", corev1beta1.KubeletConfiguration{ + Entry("evictionMaxPodGracePeriod", "evictionMaxPodGracePeriod", v1.KubeletConfiguration{ EvictionMaxPodGracePeriod: lo.ToPtr[int32](300), }), - Entry("podsPerCore", "podsPerCore", corev1beta1.KubeletConfiguration{ + Entry("podsPerCore", "podsPerCore", v1.KubeletConfiguration{ PodsPerCore: lo.ToPtr[int32](2), }), - Entry("clusterDNS", "clusterDNS", corev1beta1.KubeletConfiguration{ + Entry("clusterDNS", "clusterDNS", v1.KubeletConfiguration{ ClusterDNS: []string{"10.0.100.0"}, }), - Entry("imageGCHighThresholdPercent", "imageGCHighThresholdPercent", corev1beta1.KubeletConfiguration{ + Entry("imageGCHighThresholdPercent", "imageGCHighThresholdPercent", v1.KubeletConfiguration{ ImageGCHighThresholdPercent: lo.ToPtr[int32](50), }), - Entry("imageGCLowThresholdPercent", "imageGCLowThresholdPercent", corev1beta1.KubeletConfiguration{ + Entry("imageGCLowThresholdPercent", "imageGCLowThresholdPercent", v1.KubeletConfiguration{ ImageGCLowThresholdPercent: lo.ToPtr[int32](50), }), - Entry("cpuCFSQuota", "cpuCFSQuota", corev1beta1.KubeletConfiguration{ + Entry("cpuCFSQuota", "cpuCFSQuota", v1.KubeletConfiguration{ CPUCFSQuota: lo.ToPtr(false), }), ) }) It("should set LocalDiskStrategy to Raid0 when specified by the InstanceStorePolicy", func() { - nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1beta1.InstanceStorePolicyRAID0) + nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1.InstanceStorePolicyRAID0) ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1736,14 +1821,14 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(err).To(BeNil()) nodeClass.Spec.UserData = lo.ToPtr(string(content)) } - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) content, err := os.ReadFile("testdata/" + mergedFile) Expect(err).To(BeNil()) - expectedUserData := fmt.Sprintf(string(content), corev1beta1.NodePoolLabelKey, nodePool.Name) + expectedUserData := fmt.Sprintf(string(content), karpv1.NodePoolLabelKey, nodePool.Name) ExpectLaunchTemplatesCreatedWithUserData(expectedUserData) }, Entry("MIME", lo.ToPtr("al2023_mime_userdata_input.golden"), "al2023_mime_userdata_merged.golden"), @@ -1762,15 +1847,16 @@ var _ = Describe("LaunchTemplate Provider", func() { }) Context("Custom AMI Selector", func() { It("should use ami selector specified in EC2NodeClass", func() { - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Status.AMIs = []v1.AMI{ { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("ami-123"), - Architecture: aws.String("x86_64"), - CreationDate: aws.String("2022-08-15T12:00:00Z"), + ID: "ami-123", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureAmd64}}, + }, }, - }}) + } ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1782,16 +1868,16 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should copy over userData untouched when AMIFamily is Custom", func() { nodeClass.Spec.UserData = aws.String("special user data") - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Status.AMIs = []v1.AMI{ { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("ami-123"), - Architecture: aws.String("x86_64"), - CreationDate: aws.String("2022-08-15T12:00:00Z"), + ID: "ami-123", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureAmd64}}, + }, }, - }}) + } ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1799,20 +1885,21 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectLaunchTemplatesCreatedWithUserData("special user data") }) It("should correctly use ami selector with specific IDs in EC2NodeClass", func() { - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: "ami-123"}, {ID: "ami-456"}} + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: "ami-123"}, {ID: "ami-456"}} awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ { Name: aws.String(coretest.RandomName()), ImageId: aws.String("ami-123"), Architecture: aws.String("x86_64"), - Tags: []*ec2.Tag{{Key: aws.String(v1.LabelInstanceTypeStable), Value: aws.String("m5.large")}}, + Tags: []*ec2.Tag{{Key: aws.String(corev1.LabelInstanceTypeStable), Value: aws.String("m5.large")}}, CreationDate: aws.String("2022-08-15T12:00:00Z"), }, { Name: aws.String(coretest.RandomName()), ImageId: aws.String("ami-456"), Architecture: aws.String("x86_64"), - Tags: []*ec2.Tag{{Key: aws.String(v1.LabelInstanceTypeStable), Value: aws.String("m5.xlarge")}}, + Tags: []*ec2.Tag{{Key: aws.String(corev1.LabelInstanceTypeStable), Value: aws.String("m5.xlarge")}}, CreationDate: aws.String("2022-08-15T12:00:00Z"), }, }}) @@ -1820,6 +1907,8 @@ var _ = Describe("LaunchTemplate Provider", func() { pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) + _, err := awsEnv.AMIProvider.List(ctx, nodeClass) + Expect(err).To(BeNil()) Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(BeNumerically(">=", 2)) actualFilter := awsEnv.EC2API.CalledWithDescribeImagesInput.Pop().Filters expectedFilter := []*ec2.Filter{ @@ -1831,21 +1920,22 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(actualFilter).To(Equal(expectedFilter)) }) It("should create multiple launch templates when multiple amis are discovered with non-equivalent requirements", func() { - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Status.AMIs = []v1.AMI{ { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("ami-123"), - Architecture: aws.String("x86_64"), - CreationDate: aws.String("2022-08-15T12:00:00Z"), + ID: "ami-123", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureAmd64}}, + }, }, { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("ami-456"), - Architecture: aws.String("arm64"), - CreationDate: aws.String("2022-08-10T12:00:00Z"), + ID: "ami-456", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureArm64}}, + }, }, - }}) - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + } ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1880,14 +1970,17 @@ var _ = Describe("LaunchTemplate Provider", func() { CreationDate: aws.String("2022-01-01T12:00:00Z"), }, }}) - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} ExpectApplied(ctx, env.Client, nodeClass) - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) + ExpectObjectReconciled(ctx, env.Client, controller, nodeClass) + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, }, }, } @@ -1903,7 +1996,9 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should fail if no amis match selector.", func() { awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{}}) - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Status.AMIs = []v1.AMI{} ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1913,7 +2008,16 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should fail if no instanceType matches ami requirements.", func() { awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ {Name: aws.String(coretest.RandomName()), ImageId: aws.String("ami-123"), Architecture: aws.String("newnew"), CreationDate: aws.String("2022-01-01T12:00:00Z")}}}) - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Status.AMIs = []v1.AMI{ + { + ID: "ami-123", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{"newnew"}}, + }, + }, + } ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1925,14 +2029,14 @@ var _ = Describe("LaunchTemplate Provider", func() { awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): "test-ami-123", } - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + nodeClass.Status.AMIs = []v1.AMI{ { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("test-ami-123"), - Architecture: aws.String("x86_64"), - CreationDate: aws.String("2022-08-15T12:00:00Z"), + ID: "test-ami-123", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{string(karpv1.ArchitectureAmd64)}}, + }, }, - }}) + } ExpectApplied(ctx, env.Client, nodeClass) ExpectApplied(ctx, env.Client, nodePool) pod := coretest.UnschedulablePod() @@ -1943,38 +2047,15 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) Context("Public IP Association", func() { - It("should explicitly set 'AssociatePublicIPAddress' to false in the Launch Template", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - {Tags: map[string]string{"Name": "test-subnet-1"}}, - {Tags: map[string]string{"Name": "test-subnet-3"}}, - } - ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() - Expect(*input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress).To(BeFalse()) - }) - It("should not explicitly set 'AssociatePublicIPAddress' when the subnets are configured to assign public IPv4 addresses", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - {Tags: map[string]string{"Name": "test-subnet-2"}}, - } - ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectScheduled(ctx, env.Client, pod) - input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() - Expect(len(input.LaunchTemplateData.NetworkInterfaces)).To(BeNumerically("==", 0)) - }) DescribeTable( "should set 'AssociatePublicIPAddress' based on EC2NodeClass", func(setValue, expectedValue, isEFA bool) { nodeClass.Spec.AssociatePublicIPAddress = lo.ToPtr(setValue) ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(lo.Ternary(isEFA, coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("2")}, - Limits: v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("2")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceEFA: resource.MustParse("2")}, + Limits: corev1.ResourceList{v1.ResourceEFA: resource.MustParse("2")}, }, }, coretest.PodOptions{})) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1990,7 +2071,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) Context("Kubelet Args", func() { It("should specify the --dns-cluster-ip flag when clusterDNSIP is set", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ClusterDNS: []string{"10.0.10.100"}} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ClusterDNS: []string{"10.0.10.100"}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -2000,9 +2081,9 @@ var _ = Describe("LaunchTemplate Provider", func() { }) Context("Windows Custom UserData", func() { BeforeEach(func() { - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelOSStable, Operator: v1.NodeSelectorOpIn, Values: []string{string(v1.Windows)}}}} - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{{NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: corev1.LabelOSStable, Operator: corev1.NodeSelectorOpIn, Values: []string{string(corev1.Windows)}}}} + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "windows2022@latest"}} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} }) It("should merge and bootstrap with custom user data", func() { content, err := os.ReadFile("testdata/windows_userdata_input.golden") @@ -2012,36 +2093,36 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(nodePool), nodePool)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{ - v1.LabelOSStable: string(v1.Windows), - v1.LabelWindowsBuild: "10.0.20348", + corev1.LabelOSStable: string(corev1.Windows), + corev1.LabelWindowsBuild: "10.0.20348", }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) content, err = os.ReadFile("testdata/windows_userdata_merged.golden") Expect(err).To(BeNil()) - ExpectLaunchTemplatesCreatedWithUserData(fmt.Sprintf(string(content), corev1beta1.NodePoolLabelKey, nodePool.Name)) + ExpectLaunchTemplatesCreatedWithUserData(fmt.Sprintf(string(content), karpv1.NodePoolLabelKey, nodePool.Name)) }) It("should bootstrap when custom user data is empty", func() { ExpectApplied(ctx, env.Client, nodeClass, nodePool) Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(nodePool), nodePool)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{ - v1.LabelOSStable: string(v1.Windows), - v1.LabelWindowsBuild: "10.0.20348", + corev1.LabelOSStable: string(corev1.Windows), + corev1.LabelWindowsBuild: "10.0.20348", }, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) content, err := os.ReadFile("testdata/windows_userdata_unmerged.golden") Expect(err).To(BeNil()) - ExpectLaunchTemplatesCreatedWithUserData(fmt.Sprintf(string(content), corev1beta1.NodePoolLabelKey, nodePool.Name)) + ExpectLaunchTemplatesCreatedWithUserData(fmt.Sprintf(string(content), karpv1.NodePoolLabelKey, nodePool.Name)) }) }) }) Context("Detailed Monitoring", func() { It("should default detailed monitoring to off", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -2052,7 +2133,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should pass detailed monitoring setting to the launch template at creation", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} nodeClass.Spec.DetailedMonitoring = aws.Bool(true) ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() diff --git a/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_merged.golden index c65e73381035..335f650b12ab 100644 --- a/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_merged.golden +++ b/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_merged.golden @@ -4,6 +4,25 @@ Content-Type: multipart/mixed; boundary="//" --// Content-Type: application/node.eks.aws +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: test-cluster + clusterEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 42 +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" +--// +Content-Type: application/node.eks.aws + # Karpenter Generated NodeConfig apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig @@ -23,26 +42,10 @@ spec: clusterDNS: - 10.0.100.10 maxPods: 110 + registerWithTaints: + - effect: NoExecute + key: karpenter.sh/unregistered flags: - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" ---// -Content-Type: application/node.eks.aws - -apiVersion: node.eks.aws/v1alpha1 -kind: NodeConfig -spec: - cluster: - name: test-cluster - clusterEndpoint: https://test-cluster - certificateAuthority: cluster-ca - cidr: 10.100.0.0/16 - kubelet: - config: - maxPods: 42 ---// -Content-Type: text/x-shellscript; charset="us-ascii" - -#!/bin/bash -echo "Hello, AL2023!" --//-- diff --git a/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_merged.golden index 6dd0bb7ac6dd..2b2fdd8c3c54 100644 --- a/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_merged.golden +++ b/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_merged.golden @@ -1,6 +1,11 @@ MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="//" +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" --// Content-Type: application/node.eks.aws @@ -23,12 +28,10 @@ spec: clusterDNS: - 10.0.100.10 maxPods: 110 + registerWithTaints: + - effect: NoExecute + key: karpenter.sh/unregistered flags: - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" ---// -Content-Type: text/x-shellscript; charset="us-ascii" - -#!/bin/bash -echo "Hello, AL2023!" --//-- diff --git a/pkg/providers/launchtemplate/testdata/al2023_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/al2023_userdata_unmerged.golden index 191107c780fb..cf18ed3508f4 100644 --- a/pkg/providers/launchtemplate/testdata/al2023_userdata_unmerged.golden +++ b/pkg/providers/launchtemplate/testdata/al2023_userdata_unmerged.golden @@ -23,6 +23,9 @@ spec: clusterDNS: - 10.0.100.10 maxPods: 110 + registerWithTaints: + - effect: NoExecute + key: karpenter.sh/unregistered flags: - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" diff --git a/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_merged.golden index 724f80b5b79d..d54c3b4ece9e 100644 --- a/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_merged.golden +++ b/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_merged.golden @@ -4,6 +4,20 @@ Content-Type: multipart/mixed; boundary="//" --// Content-Type: application/node.eks.aws +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: test-cluster + clusterEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 42 +--// +Content-Type: application/node.eks.aws + # Karpenter Generated NodeConfig apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig @@ -23,21 +37,10 @@ spec: clusterDNS: - 10.0.100.10 maxPods: 110 + registerWithTaints: + - effect: NoExecute + key: karpenter.sh/unregistered flags: - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" ---// -Content-Type: application/node.eks.aws - -apiVersion: node.eks.aws/v1alpha1 -kind: NodeConfig -spec: - cluster: - name: test-cluster - clusterEndpoint: https://test-cluster - certificateAuthority: cluster-ca - cidr: 10.100.0.0/16 - kubelet: - config: - maxPods: 42 --//-- diff --git a/pkg/providers/launchtemplate/testdata/al2_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2_userdata_merged.golden index 3d1e9bc60623..22c8a37dfc03 100644 --- a/pkg/providers/launchtemplate/testdata/al2_userdata_merged.golden +++ b/pkg/providers/launchtemplate/testdata/al2_userdata_merged.golden @@ -15,5 +15,5 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 /etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ --dns-cluster-ip '10.0.100.10' \ --use-max-pods false \ ---kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" --max-pods=110' +--kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" --register-with-taints="karpenter.sh/unregistered:NoExecute" --max-pods=110' --//-- diff --git a/pkg/providers/launchtemplate/testdata/al2_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/al2_userdata_unmerged.golden index 0e9c6773d191..39ecb867db43 100644 --- a/pkg/providers/launchtemplate/testdata/al2_userdata_unmerged.golden +++ b/pkg/providers/launchtemplate/testdata/al2_userdata_unmerged.golden @@ -9,5 +9,5 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 /etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ --dns-cluster-ip '10.0.100.10' \ --use-max-pods false \ ---kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" --max-pods=110' +--kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" --register-with-taints="karpenter.sh/unregistered:NoExecute" --max-pods=110' --//-- diff --git a/pkg/providers/launchtemplate/testdata/br_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/br_userdata_merged.golden index a0f12195a192..a9a425e55aba 100644 --- a/pkg/providers/launchtemplate/testdata/br_userdata_merged.golden +++ b/pkg/providers/launchtemplate/testdata/br_userdata_merged.golden @@ -16,6 +16,7 @@ custom-node-label = 'custom' [settings.kubernetes.node-taints] baz = ['bin:NoExecute'] foo = ['bar:NoExecute'] +'karpenter.sh/unregistered' = [':NoExecute'] [settings.kubernetes.eviction-hard] 'memory.available' = '12%%' diff --git a/pkg/providers/launchtemplate/testdata/br_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/br_userdata_unmerged.golden index 67ad47bf0ebe..b81fe613e8e0 100644 --- a/pkg/providers/launchtemplate/testdata/br_userdata_unmerged.golden +++ b/pkg/providers/launchtemplate/testdata/br_userdata_unmerged.golden @@ -14,3 +14,4 @@ max-pods = 110 [settings.kubernetes.node-taints] baz = ['bin:NoExecute'] foo = ['bar:NoExecute'] +'karpenter.sh/unregistered' = [':NoExecute'] diff --git a/pkg/providers/launchtemplate/testdata/windows_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/windows_userdata_merged.golden index e1181fc5de59..d4c7b6285210 100644 --- a/pkg/providers/launchtemplate/testdata/windows_userdata_merged.golden +++ b/pkg/providers/launchtemplate/testdata/windows_userdata_merged.golden @@ -2,5 +2,5 @@ Write-Host "Running custom user data script" Write-Host "Finished running custom user data script" [string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" -& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,%s=%s,testing/cluster=unspecified" --max-pods=110' -DNSClusterIP '10.0.100.10' +& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,%s=%s,testing/cluster=unspecified" --register-with-taints="karpenter.sh/unregistered:NoExecute" --max-pods=110' -DNSClusterIP '10.0.100.10' diff --git a/pkg/providers/launchtemplate/testdata/windows_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/windows_userdata_unmerged.golden index 2c6f4b138f3b..a0b309c57f9c 100644 --- a/pkg/providers/launchtemplate/testdata/windows_userdata_unmerged.golden +++ b/pkg/providers/launchtemplate/testdata/windows_userdata_unmerged.golden @@ -1,4 +1,4 @@ [string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" -& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,%s=%s,testing/cluster=unspecified" --max-pods=110' -DNSClusterIP '10.0.100.10' +& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,%s=%s,testing/cluster=unspecified" --register-with-taints="karpenter.sh/unregistered:NoExecute" --max-pods=110' -DNSClusterIP '10.0.100.10' diff --git a/pkg/providers/pricing/pricing.go b/pkg/providers/pricing/pricing.go index 75dff1a2cc5e..b41cbbbc35ce 100644 --- a/pkg/providers/pricing/pricing.go +++ b/pkg/providers/pricing/pricing.go @@ -25,6 +25,8 @@ import ( "sync" "time" + "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/aws-sdk-go/aws" @@ -35,8 +37,6 @@ import ( "github.com/aws/aws-sdk-go/service/pricing/pricingiface" "github.com/samber/lo" "go.uber.org/multierr" - "knative.dev/pkg/logging" - "sigs.k8s.io/karpenter/pkg/utils/pretty" ) @@ -167,7 +167,7 @@ func (p *DefaultProvider) UpdateOnDemandPricing(ctx context.Context) error { // as pricing api may not be available if options.FromContext(ctx).IsolatedVPC { if p.cm.HasChanged("on-demand-prices", nil) { - logging.FromContext(ctx).Debug("running in an isolated VPC, on-demand pricing information will not be updated") + log.FromContext(ctx).V(1).Info("running in an isolated VPC, on-demand pricing information will not be updated") } return nil } @@ -221,7 +221,7 @@ func (p *DefaultProvider) UpdateOnDemandPricing(ctx context.Context) error { p.onDemandPrices = lo.Assign(onDemandPrices, onDemandMetalPrices) if p.cm.HasChanged("on-demand-prices", p.onDemandPrices) { - logging.FromContext(ctx).With("instance-type-count", len(p.onDemandPrices)).Debugf("updated on-demand pricing") + log.FromContext(ctx).WithValues("instance-type-count", len(p.onDemandPrices)).V(1).Info("updated on-demand pricing") } return nil } @@ -283,7 +283,7 @@ func (p *DefaultProvider) spotPage(ctx context.Context, prices map[string]map[st spotPrice, err := strconv.ParseFloat(spotPriceStr, 64) // these errors shouldn't occur, but if pricing API does have an error, we ignore the record if err != nil { - logging.FromContext(ctx).Debugf("unable to parse price record %#v", sph) + log.FromContext(ctx).V(1).Info(fmt.Sprintf("unable to parse price record %#v", sph)) continue } if sph.Timestamp == nil { @@ -330,12 +330,12 @@ func (p *DefaultProvider) onDemandPage(ctx context.Context, prices map[string]fl var buf bytes.Buffer enc := json.NewEncoder(&buf) if err := enc.Encode(outer); err != nil { - logging.FromContext(ctx).Errorf("encoding %s", err) + log.FromContext(ctx).Error(err, "failed encoding pricing data") } dec := json.NewDecoder(&buf) var pItem priceItem if err := dec.Decode(&pItem); err != nil { - logging.FromContext(ctx).Errorf("decoding %s", err) + log.FromContext(ctx).Error(err, "failed decoding pricing data") } if pItem.Product.Attributes.InstanceType == "" { continue @@ -393,9 +393,9 @@ func (p *DefaultProvider) UpdateSpotPricing(ctx context.Context) error { p.spotPricingUpdated = true if p.cm.HasChanged("spot-prices", p.spotPrices) { - logging.FromContext(ctx).With( + log.FromContext(ctx).WithValues( "instance-type-count", len(p.onDemandPrices), - "offering-count", totalOfferings).Debugf("updated spot pricing with instance types and offerings") + "offering-count", totalOfferings).V(1).Info("updated spot pricing with instance types and offerings") } return nil } diff --git a/pkg/providers/pricing/zz_generated.pricing_aws.go b/pkg/providers/pricing/zz_generated.pricing_aws.go index 91bce11f094d..d729268b5d82 100644 --- a/pkg/providers/pricing/zz_generated.pricing_aws.go +++ b/pkg/providers/pricing/zz_generated.pricing_aws.go @@ -16,7 +16,7 @@ limitations under the License. package pricing -// generated at 2024-03-11T13:05:10Z for us-east-1 +// generated at 2024-07-10T14:30:58Z for us-east-1 var InitialOnDemandPricesAWS = map[string]map[string]float64{ // us-east-1 @@ -90,7 +90,8 @@ var InitialOnDemandPricesAWS = map[string]map[string]float64{ "c7gd.xlarge": 0.181400, // c7gn family "c7gn.12xlarge": 2.995200, "c7gn.16xlarge": 3.993600, "c7gn.2xlarge": 0.499200, "c7gn.4xlarge": 0.998400, - "c7gn.8xlarge": 1.996800, "c7gn.large": 0.124800, "c7gn.medium": 0.062400, "c7gn.xlarge": 0.249600, + "c7gn.8xlarge": 1.996800, "c7gn.large": 0.124800, "c7gn.medium": 0.062400, "c7gn.metal": 3.993600, + "c7gn.xlarge": 0.249600, // c7i family "c7i.12xlarge": 2.142000, "c7i.16xlarge": 2.856000, "c7i.24xlarge": 4.284000, "c7i.2xlarge": 0.357000, "c7i.48xlarge": 8.568000, "c7i.4xlarge": 0.714000, "c7i.8xlarge": 1.428000, "c7i.large": 0.089250, @@ -126,6 +127,11 @@ var InitialOnDemandPricesAWS = map[string]map[string]float64{ // g5g family "g5g.16xlarge": 2.744000, "g5g.2xlarge": 0.556000, "g5g.4xlarge": 0.828000, "g5g.8xlarge": 1.372000, "g5g.metal": 2.744000, "g5g.xlarge": 0.420000, + // g6 family + "g6.12xlarge": 4.601600, "g6.16xlarge": 3.396800, "g6.24xlarge": 6.675200, "g6.2xlarge": 0.977600, + "g6.48xlarge": 13.350400, "g6.4xlarge": 1.323200, "g6.8xlarge": 2.014400, "g6.xlarge": 0.804800, + // gr6 family + "gr6.4xlarge": 1.539200, "gr6.8xlarge": 2.446400, // h1 family "h1.16xlarge": 3.744000, "h1.2xlarge": 0.468000, "h1.4xlarge": 0.936000, "h1.8xlarge": 1.872000, // hpc7g family @@ -328,6 +334,10 @@ var InitialOnDemandPricesAWS = map[string]map[string]float64{ "r7iz.12xlarge": 4.464000, "r7iz.16xlarge": 5.952000, "r7iz.2xlarge": 0.744000, "r7iz.32xlarge": 11.904000, "r7iz.4xlarge": 1.488000, "r7iz.8xlarge": 2.976000, "r7iz.large": 0.186000, "r7iz.metal-16xl": 6.547200, "r7iz.metal-32xl": 13.094400, "r7iz.xlarge": 0.372000, + // r8g family + "r8g.12xlarge": 2.827680, "r8g.16xlarge": 3.770240, "r8g.24xlarge": 5.655360, "r8g.2xlarge": 0.471280, + "r8g.48xlarge": 11.310720, "r8g.4xlarge": 0.942560, "r8g.8xlarge": 1.885120, "r8g.large": 0.117820, + "r8g.medium": 0.058910, "r8g.metal-24xl": 6.220900, "r8g.metal-48xl": 11.310720, "r8g.xlarge": 0.235640, // t1 family "t1.micro": 0.020000, // t2 family @@ -358,6 +368,14 @@ var InitialOnDemandPricesAWS = map[string]map[string]float64{ "u-6tb1.112xlarge": 54.600000, "u-6tb1.56xlarge": 46.403910, // u-9tb1 family "u-9tb1.112xlarge": 81.900000, + // u7i-12tb family + "u7i-12tb.224xlarge": 152.880000, + // u7in-16tb family + "u7in-16tb.224xlarge": 203.840000, + // u7in-24tb family + "u7in-24tb.224xlarge": 305.760000, + // u7in-32tb family + "u7in-32tb.224xlarge": 407.680000, // vt1 family "vt1.24xlarge": 5.200000, "vt1.3xlarge": 0.650000, "vt1.6xlarge": 1.300000, // x1 family diff --git a/pkg/providers/pricing/zz_generated.pricing_aws_us_gov.go b/pkg/providers/pricing/zz_generated.pricing_aws_us_gov.go index 099208bd90d9..35aa66e3c110 100644 --- a/pkg/providers/pricing/zz_generated.pricing_aws_us_gov.go +++ b/pkg/providers/pricing/zz_generated.pricing_aws_us_gov.go @@ -16,7 +16,7 @@ limitations under the License. package pricing -// generated at 2024-03-18T13:06:23Z for us-east-1 +// generated at 2024-05-27T13:07:14Z for us-east-1 var InitialOnDemandPricesUSGov = map[string]map[string]float64{ // us-gov-east-1 @@ -53,6 +53,10 @@ var InitialOnDemandPricesUSGov = map[string]map[string]float64{ "c6in.12xlarge": 3.276000, "c6in.16xlarge": 4.368000, "c6in.24xlarge": 6.552000, "c6in.2xlarge": 0.546000, "c6in.32xlarge": 8.736000, "c6in.4xlarge": 1.092000, "c6in.8xlarge": 2.184000, "c6in.large": 0.136500, "c6in.metal": 8.736000, "c6in.xlarge": 0.273000, + // c7i family + "c7i.12xlarge": 2.570400, "c7i.16xlarge": 3.427200, "c7i.24xlarge": 5.140800, "c7i.2xlarge": 0.428400, + "c7i.48xlarge": 10.281600, "c7i.4xlarge": 0.856800, "c7i.8xlarge": 1.713600, "c7i.large": 0.107100, + "c7i.metal-24xl": 5.654880, "c7i.metal-48xl": 10.281600, "c7i.xlarge": 0.214200, // d2 family "d2.2xlarge": 1.656000, "d2.4xlarge": 3.312000, "d2.8xlarge": 6.624000, "d2.xlarge": 0.828000, // g4dn family @@ -74,7 +78,8 @@ var InitialOnDemandPricesUSGov = map[string]map[string]float64{ "inf1.24xlarge": 5.953000, "inf1.2xlarge": 0.456000, "inf1.6xlarge": 1.488000, "inf1.xlarge": 0.288000, // m5 family "m5.12xlarge": 2.904000, "m5.16xlarge": 3.872000, "m5.24xlarge": 5.808000, "m5.2xlarge": 0.484000, - "m5.4xlarge": 0.968000, "m5.8xlarge": 1.936000, "m5.large": 0.121000, "m5.xlarge": 0.242000, + "m5.4xlarge": 0.968000, "m5.8xlarge": 1.936000, "m5.large": 0.121000, "m5.metal": 5.808000, + "m5.xlarge": 0.242000, // m5a family "m5a.12xlarge": 2.616000, "m5a.16xlarge": 3.488000, "m5a.24xlarge": 5.232000, "m5a.2xlarge": 0.436000, "m5a.4xlarge": 0.872000, "m5a.8xlarge": 1.744000, "m5a.large": 0.109000, "m5a.xlarge": 0.218000, @@ -102,6 +107,13 @@ var InitialOnDemandPricesUSGov = map[string]map[string]float64{ "m6i.12xlarge": 2.904000, "m6i.16xlarge": 3.872000, "m6i.24xlarge": 5.808000, "m6i.2xlarge": 0.484000, "m6i.32xlarge": 7.744000, "m6i.4xlarge": 0.968000, "m6i.8xlarge": 1.936000, "m6i.large": 0.121000, "m6i.metal": 7.744000, "m6i.xlarge": 0.242000, + // m7i-flex family + "m7i-flex.2xlarge": 0.482800, "m7i-flex.4xlarge": 0.965600, "m7i-flex.8xlarge": 1.931200, + "m7i-flex.large": 0.120700, "m7i-flex.xlarge": 0.241400, + // m7i family + "m7i.12xlarge": 3.049200, "m7i.16xlarge": 4.065600, "m7i.24xlarge": 6.098400, "m7i.2xlarge": 0.508200, + "m7i.48xlarge": 12.196800, "m7i.4xlarge": 1.016400, "m7i.8xlarge": 2.032800, "m7i.large": 0.127050, + "m7i.metal-24xl": 6.708240, "m7i.metal-48xl": 12.196800, "m7i.xlarge": 0.254100, // p3dn family "p3dn.24xlarge": 37.454000, // r5 family @@ -135,6 +147,10 @@ var InitialOnDemandPricesUSGov = map[string]map[string]float64{ "r6i.12xlarge": 3.624000, "r6i.16xlarge": 4.832000, "r6i.24xlarge": 7.248000, "r6i.2xlarge": 0.604000, "r6i.32xlarge": 9.664000, "r6i.4xlarge": 1.208000, "r6i.8xlarge": 2.416000, "r6i.large": 0.151000, "r6i.metal": 9.664000, "r6i.xlarge": 0.302000, + // r7i family + "r7i.12xlarge": 3.805200, "r7i.16xlarge": 5.073600, "r7i.24xlarge": 7.610400, "r7i.2xlarge": 0.634200, + "r7i.48xlarge": 15.220800, "r7i.4xlarge": 1.268400, "r7i.8xlarge": 2.536800, "r7i.large": 0.158550, + "r7i.metal-24xl": 8.371440, "r7i.metal-48xl": 15.220800, "r7i.xlarge": 0.317100, // t3 family "t3.2xlarge": 0.390400, "t3.large": 0.097600, "t3.medium": 0.048800, "t3.micro": 0.012200, "t3.nano": 0.006100, "t3.small": 0.024400, "t3.xlarge": 0.195200, @@ -310,6 +326,13 @@ var InitialOnDemandPricesUSGov = map[string]map[string]float64{ "m6in.12xlarge": 4.183920, "m6in.16xlarge": 5.578560, "m6in.24xlarge": 8.367840, "m6in.2xlarge": 0.697320, "m6in.32xlarge": 11.157120, "m6in.4xlarge": 1.394640, "m6in.8xlarge": 2.789280, "m6in.large": 0.174330, "m6in.metal": 11.157120, "m6in.xlarge": 0.348660, + // m7i-flex family + "m7i-flex.2xlarge": 0.482800, "m7i-flex.4xlarge": 0.965600, "m7i-flex.8xlarge": 1.931200, + "m7i-flex.large": 0.120700, "m7i-flex.xlarge": 0.241400, + // m7i family + "m7i.12xlarge": 3.049200, "m7i.16xlarge": 4.065600, "m7i.24xlarge": 6.098400, "m7i.2xlarge": 0.508200, + "m7i.48xlarge": 12.196800, "m7i.4xlarge": 1.016400, "m7i.8xlarge": 2.032800, "m7i.large": 0.127050, + "m7i.metal-24xl": 6.708240, "m7i.metal-48xl": 12.196800, "m7i.xlarge": 0.254100, // p2 family "p2.16xlarge": 17.280000, "p2.8xlarge": 8.640000, "p2.xlarge": 1.080000, // p3 family @@ -370,6 +393,10 @@ var InitialOnDemandPricesUSGov = map[string]map[string]float64{ "r6in.12xlarge": 5.026320, "r6in.16xlarge": 6.701760, "r6in.24xlarge": 10.052640, "r6in.2xlarge": 0.837720, "r6in.32xlarge": 13.403520, "r6in.4xlarge": 1.675440, "r6in.8xlarge": 3.350880, "r6in.large": 0.209430, "r6in.metal": 13.403520, "r6in.xlarge": 0.418860, + // r7i family + "r7i.12xlarge": 3.805200, "r7i.16xlarge": 5.073600, "r7i.24xlarge": 7.610400, "r7i.2xlarge": 0.634200, + "r7i.48xlarge": 15.220800, "r7i.4xlarge": 1.268400, "r7i.8xlarge": 2.536800, "r7i.large": 0.158550, + "r7i.metal-24xl": 8.371440, "r7i.metal-48xl": 15.220800, "r7i.xlarge": 0.317100, // t1 family "t1.micro": 0.024000, // t2 family diff --git a/pkg/providers/securitygroup/securitygroup.go b/pkg/providers/securitygroup/securitygroup.go index 94a89ad20675..998498277f32 100644 --- a/pkg/providers/securitygroup/securitygroup.go +++ b/pkg/providers/securitygroup/securitygroup.go @@ -25,15 +25,15 @@ import ( "github.com/mitchellh/hashstructure/v2" "github.com/patrickmn/go-cache" "github.com/samber/lo" - "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/karpenter/pkg/utils/pretty" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" ) type Provider interface { - List(context.Context, *v1beta1.EC2NodeClass) ([]*ec2.SecurityGroup, error) + List(context.Context, *v1.EC2NodeClass) ([]*ec2.SecurityGroup, error) } type DefaultProvider struct { @@ -52,7 +52,7 @@ func NewDefaultProvider(ec2api ec2iface.EC2API, cache *cache.Cache) *DefaultProv } } -func (p *DefaultProvider) List(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) ([]*ec2.SecurityGroup, error) { +func (p *DefaultProvider) List(ctx context.Context, nodeClass *v1.EC2NodeClass) ([]*ec2.SecurityGroup, error) { p.Lock() defer p.Unlock() @@ -62,12 +62,11 @@ func (p *DefaultProvider) List(ctx context.Context, nodeClass *v1beta1.EC2NodeCl if err != nil { return nil, err } - if p.cm.HasChanged(fmt.Sprintf("security-groups/%s", nodeClass.Name), securityGroups) { - logging.FromContext(ctx). - With("security-groups", lo.Map(securityGroups, func(s *ec2.SecurityGroup, _ int) string { - return aws.StringValue(s.GroupId) - })). - Debugf("discovered security groups") + securityGroupIDs := lo.Map(securityGroups, func(s *ec2.SecurityGroup, _ int) string { return aws.StringValue(s.GroupId) }) + if p.cm.HasChanged(fmt.Sprintf("security-groups/%s", nodeClass.Name), securityGroupIDs) { + log.FromContext(ctx). + WithValues("security-groups", securityGroupIDs). + V(1).Info("discovered security groups") } return securityGroups, nil } @@ -78,7 +77,9 @@ func (p *DefaultProvider) getSecurityGroups(ctx context.Context, filterSets [][] return nil, err } if sg, ok := p.cache.Get(fmt.Sprint(hash)); ok { - return sg.([]*ec2.SecurityGroup), nil + // Ensure what's returned from this function is a shallow-copy of the slice (not a deep-copy of the data itself) + // so that modifications to the ordering of the data don't affect the original + return append([]*ec2.SecurityGroup{}, sg.([]*ec2.SecurityGroup)...), nil } securityGroups := map[string]*ec2.SecurityGroup{} for _, filters := range filterSets { @@ -94,7 +95,7 @@ func (p *DefaultProvider) getSecurityGroups(ctx context.Context, filterSets [][] return lo.Values(securityGroups), nil } -func getFilterSets(terms []v1beta1.SecurityGroupSelectorTerm) (res [][]*ec2.Filter) { +func getFilterSets(terms []v1.SecurityGroupSelectorTerm) (res [][]*ec2.Filter) { idFilter := &ec2.Filter{Name: aws.String("group-id")} nameFilter := &ec2.Filter{Name: aws.String("group-name")} for _, term := range terms { diff --git a/pkg/providers/securitygroup/suite_test.go b/pkg/providers/securitygroup/suite_test.go index 901dbcdd5d35..6a2e18fe8111 100644 --- a/pkg/providers/securitygroup/suite_test.go +++ b/pkg/providers/securitygroup/suite_test.go @@ -16,32 +16,35 @@ package securitygroup_test import ( "context" + "sort" + "sync" "testing" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/samber/lo" "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/test" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) var ctx context.Context var stop context.CancelFunc var env *coretest.Environment var awsEnv *test.Environment -var nodeClass *v1beta1.EC2NodeClass +var nodeClass *v1.EC2NodeClass func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -50,7 +53,7 @@ func TestAWS(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) ctx, stop = context.WithCancel(ctx) @@ -65,17 +68,19 @@ var _ = AfterSuite(func() { var _ = BeforeEach(func() { ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) - nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ - AMIFamily: aws.String(v1beta1.AMIFamilyAL2), - SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ + nodeClass = test.EC2NodeClass(v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: []v1.AMISelectorTerm{{ + Alias: "al2@latest", + }}, + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ { Tags: map[string]string{ "*": "*", }, }, }, - SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{ "*": "*", @@ -129,7 +134,7 @@ var _ = Describe("SecurityGroupProvider", func() { }, securityGroups) }) It("should discover security groups by multiple tag values", func() { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"Name": "test-security-group-1"}, }, @@ -151,7 +156,7 @@ var _ = Describe("SecurityGroupProvider", func() { }, securityGroups) }) It("should discover security groups by ID", func() { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { ID: "sg-test1", }, @@ -166,7 +171,7 @@ var _ = Describe("SecurityGroupProvider", func() { }, securityGroups) }) It("should discover security groups by IDs", func() { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { ID: "sg-test1", }, @@ -188,7 +193,7 @@ var _ = Describe("SecurityGroupProvider", func() { }, securityGroups) }) It("should discover security groups by IDs and tags", func() { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { ID: "sg-test1", Tags: map[string]string{"foo": "bar"}, @@ -212,7 +217,7 @@ var _ = Describe("SecurityGroupProvider", func() { }, securityGroups) }) It("should discover security groups by IDs intersected with tags", func() { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { ID: "sg-test2", Tags: map[string]string{"foo": "bar"}, @@ -228,7 +233,7 @@ var _ = Describe("SecurityGroupProvider", func() { }, securityGroups) }) It("should discover security groups by names", func() { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { Name: "securityGroup-test2", }, @@ -250,7 +255,7 @@ var _ = Describe("SecurityGroupProvider", func() { }, securityGroups) }) It("should discover security groups by names intersected with tags", func() { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { Name: "securityGroup-test3", Tags: map[string]string{"TestTag": "*"}, @@ -269,7 +274,7 @@ var _ = Describe("SecurityGroupProvider", func() { It("should resolve security groups from cache that are filtered by id", func() { expectedSecurityGroups := awsEnv.EC2API.DescribeSecurityGroupsOutput.Clone().SecurityGroups for _, sg := range expectedSecurityGroups { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { ID: *sg.GroupId, }, @@ -288,7 +293,7 @@ var _ = Describe("SecurityGroupProvider", func() { It("should resolve security groups from cache that are filtered by Name", func() { expectedSecurityGroups := awsEnv.EC2API.DescribeSecurityGroupsOutput.Clone().SecurityGroups for _, sg := range expectedSecurityGroups { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { Name: *sg.GroupName, }, @@ -313,7 +318,7 @@ var _ = Describe("SecurityGroupProvider", func() { return map[string]string{"Name": lo.FromPtr(tag.Value)} }) for _, tag := range tagSet { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { Tags: tag, }, @@ -330,6 +335,72 @@ var _ = Describe("SecurityGroupProvider", func() { } }) }) + It("should not cause data races when calling List() simultaneously", func() { + wg := sync.WaitGroup{} + for i := 0; i < 10000; i++ { + wg.Add(1) + go func() { + defer wg.Done() + defer GinkgoRecover() + securityGroups, err := awsEnv.SecurityGroupProvider.List(ctx, nodeClass) + Expect(err).ToNot(HaveOccurred()) + + Expect(securityGroups).To(HaveLen(3)) + // Sort everything in parallel and ensure that we don't get data races + sort.Slice(securityGroups, func(i, j int) bool { + return *securityGroups[i].GroupId < *securityGroups[j].GroupId + }) + Expect(securityGroups).To(BeEquivalentTo([]*ec2.SecurityGroup{ + { + GroupId: lo.ToPtr("sg-test1"), + GroupName: lo.ToPtr("securityGroup-test1"), + Tags: []*ec2.Tag{ + { + Key: lo.ToPtr("Name"), + Value: lo.ToPtr("test-security-group-1"), + }, + { + Key: lo.ToPtr("foo"), + Value: lo.ToPtr("bar"), + }, + }, + }, + { + GroupId: lo.ToPtr("sg-test2"), + GroupName: lo.ToPtr("securityGroup-test2"), + Tags: []*ec2.Tag{ + { + Key: lo.ToPtr("Name"), + Value: lo.ToPtr("test-security-group-2"), + }, + { + Key: lo.ToPtr("foo"), + Value: lo.ToPtr("bar"), + }, + }, + }, + { + GroupId: lo.ToPtr("sg-test3"), + GroupName: lo.ToPtr("securityGroup-test3"), + Tags: []*ec2.Tag{ + { + Key: lo.ToPtr("Name"), + Value: lo.ToPtr("test-security-group-3"), + }, + { + Key: lo.ToPtr("TestTag"), + }, + { + Key: lo.ToPtr("foo"), + Value: lo.ToPtr("bar"), + }, + }, + }, + })) + }() + } + wg.Wait() + }) }) func ExpectConsistsOfSecurityGroups(expected, actual []*ec2.SecurityGroup) { diff --git a/pkg/providers/ssm/provider.go b/pkg/providers/ssm/provider.go new file mode 100644 index 000000000000..586a13bf47a3 --- /dev/null +++ b/pkg/providers/ssm/provider.go @@ -0,0 +1,61 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ssm + +import ( + "context" + "fmt" + "sync" + + "github.com/aws/aws-sdk-go/service/ssm" + "github.com/aws/aws-sdk-go/service/ssm/ssmiface" + "github.com/patrickmn/go-cache" + "github.com/samber/lo" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type Provider interface { + Get(context.Context, string) (string, error) +} + +type DefaultProvider struct { + sync.Mutex + cache *cache.Cache + ssmapi ssmiface.SSMAPI +} + +func NewDefaultProvider(ssmapi ssmiface.SSMAPI, cache *cache.Cache) *DefaultProvider { + return &DefaultProvider{ + ssmapi: ssmapi, + cache: cache, + } +} + +func (p *DefaultProvider) Get(ctx context.Context, parameter string) (string, error) { + p.Lock() + defer p.Unlock() + if result, ok := p.cache.Get(parameter); ok { + return result.(string), nil + } + result, err := p.ssmapi.GetParameterWithContext(ctx, &ssm.GetParameterInput{ + Name: lo.ToPtr(parameter), + }) + if err != nil { + return "", fmt.Errorf("getting ssm parameter %q, %w", parameter, err) + } + p.cache.SetDefault(parameter, lo.FromPtr(result.Parameter.Value)) + log.FromContext(ctx).WithValues("parameter", parameter, "value", result.Parameter.Value).Info("discovered ssm parameter") + return lo.FromPtr(result.Parameter.Value), nil +} diff --git a/pkg/providers/subnet/subnet.go b/pkg/providers/subnet/subnet.go index 37af40718b25..58fc1776804c 100644 --- a/pkg/providers/subnet/subnet.go +++ b/pkg/providers/subnet/subnet.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "net/http" - "sort" "sync" "github.com/aws/aws-sdk-go/aws" @@ -27,43 +26,56 @@ import ( "github.com/mitchellh/hashstructure/v2" "github.com/patrickmn/go-cache" "github.com/samber/lo" - "knative.dev/pkg/logging" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/log" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" + "sigs.k8s.io/karpenter/pkg/scheduling" "sigs.k8s.io/karpenter/pkg/utils/pretty" ) type Provider interface { LivenessProbe(*http.Request) error - List(context.Context, *v1beta1.EC2NodeClass) ([]*ec2.Subnet, error) - CheckAnyPublicIPAssociations(context.Context, *v1beta1.EC2NodeClass) (bool, error) - ZonalSubnetsForLaunch(context.Context, *v1beta1.EC2NodeClass, []*cloudprovider.InstanceType, string) (map[string]*ec2.Subnet, error) - UpdateInflightIPs(*ec2.CreateFleetInput, *ec2.CreateFleetOutput, []*cloudprovider.InstanceType, []*ec2.Subnet, string) + List(context.Context, *v1.EC2NodeClass) ([]*ec2.Subnet, error) + ZonalSubnetsForLaunch(context.Context, *v1.EC2NodeClass, []*cloudprovider.InstanceType, string) (map[string]*Subnet, error) + UpdateInflightIPs(*ec2.CreateFleetInput, *ec2.CreateFleetOutput, []*cloudprovider.InstanceType, []*Subnet, string) } type DefaultProvider struct { - sync.RWMutex - ec2api ec2iface.EC2API - cache *cache.Cache - cm *pretty.ChangeMonitor - inflightIPs map[string]int64 + sync.Mutex + ec2api ec2iface.EC2API + cache *cache.Cache + availableIPAddressCache *cache.Cache + associatePublicIPAddressCache *cache.Cache + cm *pretty.ChangeMonitor + inflightIPs map[string]int64 } -func NewDefaultProvider(ec2api ec2iface.EC2API, cache *cache.Cache) *DefaultProvider { +type Subnet struct { + ID string + Zone string + ZoneID string + AvailableIPAddressCount int64 +} + +func NewDefaultProvider(ec2api ec2iface.EC2API, cache *cache.Cache, availableIPAddressCache *cache.Cache, associatePublicIPAddressCache *cache.Cache) *DefaultProvider { return &DefaultProvider{ ec2api: ec2api, cm: pretty.NewChangeMonitor(), // TODO: Remove cache when we utilize the resolved subnets from the EC2NodeClass.status // Subnets are sorted on AvailableIpAddressCount, descending order - cache: cache, + cache: cache, + availableIPAddressCache: availableIPAddressCache, + associatePublicIPAddressCache: associatePublicIPAddressCache, // inflightIPs is used to track IPs from known launched instances inflightIPs: map[string]int64{}, } } -func (p *DefaultProvider) List(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) ([]*ec2.Subnet, error) { +func (p *DefaultProvider) List(ctx context.Context, nodeClass *v1.EC2NodeClass) ([]*ec2.Subnet, error) { p.Lock() defer p.Unlock() filterSets := getFilterSets(nodeClass.Spec.SubnetSelectorTerms) @@ -75,7 +87,9 @@ func (p *DefaultProvider) List(ctx context.Context, nodeClass *v1beta1.EC2NodeCl return nil, err } if subnets, ok := p.cache.Get(fmt.Sprint(hash)); ok { - return subnets.([]*ec2.Subnet), nil + // Ensure what's returned from this function is a shallow-copy of the slice (not a deep-copy of the data itself) + // so that modifications to the ordering of the data don't affect the original + return append([]*ec2.Subnet{}, subnets.([]*ec2.Subnet)...), nil } // Ensure that all the subnets that are returned here are unique @@ -87,74 +101,79 @@ func (p *DefaultProvider) List(ctx context.Context, nodeClass *v1beta1.EC2NodeCl } for i := range output.Subnets { subnets[lo.FromPtr(output.Subnets[i].SubnetId)] = output.Subnets[i] + p.availableIPAddressCache.SetDefault(lo.FromPtr(output.Subnets[i].SubnetId), lo.FromPtr(output.Subnets[i].AvailableIpAddressCount)) + p.associatePublicIPAddressCache.SetDefault(lo.FromPtr(output.Subnets[i].SubnetId), lo.FromPtr(output.Subnets[i].MapPublicIpOnLaunch)) + // subnets can be leaked here, if a subnets is never called received from ec2 + // we are accepting it for now, as this will be an insignificant amount of memory delete(p.inflightIPs, lo.FromPtr(output.Subnets[i].SubnetId)) // remove any previously tracked IP addresses since we just refreshed from EC2 } } p.cache.SetDefault(fmt.Sprint(hash), lo.Values(subnets)) - if p.cm.HasChanged(fmt.Sprintf("subnets/%s", nodeClass.Name), subnets) { - logging.FromContext(ctx). - With("subnets", lo.Map(lo.Values(subnets), func(s *ec2.Subnet, _ int) string { - return fmt.Sprintf("%s (%s)", aws.StringValue(s.SubnetId), aws.StringValue(s.AvailabilityZone)) - })). - Debugf("discovered subnets") + if p.cm.HasChanged(fmt.Sprintf("subnets/%s", nodeClass.Name), lo.Keys(subnets)) { + log.FromContext(ctx). + WithValues("subnets", lo.Map(lo.Values(subnets), func(s *ec2.Subnet, _ int) v1.Subnet { + return v1.Subnet{ + ID: lo.FromPtr(s.SubnetId), + Zone: lo.FromPtr(s.AvailabilityZone), + ZoneID: lo.FromPtr(s.AvailabilityZoneId), + } + })).V(1).Info("discovered subnets") } return lo.Values(subnets), nil } -// CheckAnyPublicIPAssociations returns a bool indicating whether all referenced subnets assign public IPv4 addresses to EC2 instances created therein -func (p *DefaultProvider) CheckAnyPublicIPAssociations(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (bool, error) { - subnets, err := p.List(ctx, nodeClass) - if err != nil { - return false, err - } - _, ok := lo.Find(subnets, func(s *ec2.Subnet) bool { - return aws.BoolValue(s.MapPublicIpOnLaunch) - }) - return ok, nil -} - // ZonalSubnetsForLaunch returns a mapping of zone to the subnet with the most available IP addresses and deducts the passed ips from the available count -func (p *DefaultProvider) ZonalSubnetsForLaunch(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, instanceTypes []*cloudprovider.InstanceType, capacityType string) (map[string]*ec2.Subnet, error) { - subnets, err := p.List(ctx, nodeClass) - if err != nil { - return nil, err - } - if len(subnets) == 0 { +func (p *DefaultProvider) ZonalSubnetsForLaunch(ctx context.Context, nodeClass *v1.EC2NodeClass, instanceTypes []*cloudprovider.InstanceType, capacityType string) (map[string]*Subnet, error) { + if len(nodeClass.Status.Subnets) == 0 { return nil, fmt.Errorf("no subnets matched selector %v", nodeClass.Spec.SubnetSelectorTerms) } + p.Lock() defer p.Unlock() - // sort subnets in ascending order of available IP addresses and populate map with most available subnet per AZ - zonalSubnets := map[string]*ec2.Subnet{} - sort.Slice(subnets, func(i, j int) bool { - iIPs := aws.Int64Value(subnets[i].AvailableIpAddressCount) - jIPs := aws.Int64Value(subnets[j].AvailableIpAddressCount) - // override ip count from ec2.Subnet if we've tracked launches - if ips, ok := p.inflightIPs[*subnets[i].SubnetId]; ok { - iIPs = ips + + zonalSubnets := map[string]*Subnet{} + availableIPAddressCount := map[string]int64{} + for _, subnet := range nodeClass.Status.Subnets { + if subnetAvailableIP, ok := p.availableIPAddressCache.Get(subnet.ID); ok { + availableIPAddressCount[subnet.ID] = subnetAvailableIP.(int64) } - if ips, ok := p.inflightIPs[*subnets[j].SubnetId]; ok { - jIPs = ips + } + + for _, subnet := range nodeClass.Status.Subnets { + if v, ok := zonalSubnets[subnet.Zone]; ok { + currentZonalSubnetIPAddressCount := v.AvailableIPAddressCount + newZonalSubnetIPAddressCount := availableIPAddressCount[subnet.ID] + if ips, ok := p.inflightIPs[v.ID]; ok { + currentZonalSubnetIPAddressCount = ips + } + if ips, ok := p.inflightIPs[subnet.ID]; ok { + newZonalSubnetIPAddressCount = ips + } + + if currentZonalSubnetIPAddressCount >= newZonalSubnetIPAddressCount { + continue + } } - return iIPs < jIPs - }) - for _, subnet := range subnets { - zonalSubnets[*subnet.AvailabilityZone] = subnet + zonalSubnets[subnet.Zone] = &Subnet{ID: subnet.ID, Zone: subnet.Zone, ZoneID: subnet.ZoneID, AvailableIPAddressCount: availableIPAddressCount[subnet.ID]} } + for _, subnet := range zonalSubnets { - predictedIPsUsed := p.minPods(instanceTypes, *subnet.AvailabilityZone, capacityType) - prevIPs := *subnet.AvailableIpAddressCount - if trackedIPs, ok := p.inflightIPs[*subnet.SubnetId]; ok { + predictedIPsUsed := p.minPods(instanceTypes, scheduling.NewRequirements( + scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, capacityType), + scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, subnet.Zone), + )) + prevIPs := subnet.AvailableIPAddressCount + if trackedIPs, ok := p.inflightIPs[subnet.ID]; ok { prevIPs = trackedIPs } - p.inflightIPs[*subnet.SubnetId] = prevIPs - predictedIPsUsed + p.inflightIPs[subnet.ID] = prevIPs - predictedIPsUsed } return zonalSubnets, nil } // UpdateInflightIPs is used to refresh the in-memory IP usage by adding back unused IPs after a CreateFleet response is returned func (p *DefaultProvider) UpdateInflightIPs(createFleetInput *ec2.CreateFleetInput, createFleetOutput *ec2.CreateFleetOutput, instanceTypes []*cloudprovider.InstanceType, - subnets []*ec2.Subnet, capacityType string) { + subnets []*Subnet, capacityType string) { p.Lock() defer p.Unlock() @@ -182,30 +201,33 @@ func (p *DefaultProvider) UpdateInflightIPs(createFleetInput *ec2.CreateFleetInp // Find the subnets that were included in the input but not chosen by Fleet, so we need to add the inflight IPs back to them subnetIDsToAddBackIPs, _ := lo.Difference(fleetInputSubnets, fleetOutputSubnets) - // Aggregate all the cached subnets - cachedSubnets := lo.UniqBy(lo.Flatten(lo.MapToSlice(p.cache.Items(), func(_ string, item cache.Item) []*ec2.Subnet { - return item.Object.([]*ec2.Subnet) - })), func(subnet *ec2.Subnet) string { return *subnet.SubnetId }) + // Aggregate all the cached subnets ip address count + cachedAvailableIPAddressMap := lo.MapEntries(p.availableIPAddressCache.Items(), func(k string, v cache.Item) (string, int64) { + return k, v.Object.(int64) + }) // Update the inflight IP tracking of subnets stored in the cache that have not be synchronized since the initial // deduction of IP addresses before the instance launch - for _, cachedSubnet := range cachedSubnets { - if !lo.Contains(subnetIDsToAddBackIPs, *cachedSubnet.SubnetId) { + for cachedSubnetID, cachedIPAddressCount := range cachedAvailableIPAddressMap { + if !lo.Contains(subnetIDsToAddBackIPs, cachedSubnetID) { continue } - originalSubnet, ok := lo.Find(subnets, func(subnet *ec2.Subnet) bool { - return *subnet.SubnetId == *cachedSubnet.SubnetId + originalSubnet, ok := lo.Find(subnets, func(subnet *Subnet) bool { + return subnet.ID == cachedSubnetID }) if !ok { continue } // If the cached subnet IP address count hasn't changed from the original subnet used to // launch the instance, then we need to update the tracked IPs - if *originalSubnet.AvailableIpAddressCount == *cachedSubnet.AvailableIpAddressCount { + if originalSubnet.AvailableIPAddressCount == cachedIPAddressCount { // other IPs deducted were opportunistic and need to be readded since Fleet didn't pick those subnets to launch into - if ips, ok := p.inflightIPs[*originalSubnet.SubnetId]; ok { - minPods := p.minPods(instanceTypes, *originalSubnet.AvailabilityZone, capacityType) - p.inflightIPs[*originalSubnet.SubnetId] = ips + minPods + if ips, ok := p.inflightIPs[originalSubnet.ID]; ok { + minPods := p.minPods(instanceTypes, scheduling.NewRequirements( + scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, capacityType), + scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, originalSubnet.Zone), + )) + p.inflightIPs[originalSubnet.ID] = ips + minPods } } } @@ -218,14 +240,10 @@ func (p *DefaultProvider) LivenessProbe(_ *http.Request) error { return nil } -func (p *DefaultProvider) minPods(instanceTypes []*cloudprovider.InstanceType, zone string, capacityType string) int64 { +func (p *DefaultProvider) minPods(instanceTypes []*cloudprovider.InstanceType, reqs scheduling.Requirements) int64 { // filter for instance types available in the zone and capacity type being requested filteredInstanceTypes := lo.Filter(instanceTypes, func(it *cloudprovider.InstanceType, _ int) bool { - offering, ok := it.Offerings.Get(capacityType, zone) - if !ok { - return false - } - return offering.Available + return it.Offerings.Available().HasCompatible(reqs) }) if len(filteredInstanceTypes) == 0 { return 0 @@ -237,7 +255,7 @@ func (p *DefaultProvider) minPods(instanceTypes []*cloudprovider.InstanceType, z return pods } -func getFilterSets(terms []v1beta1.SubnetSelectorTerm) (res [][]*ec2.Filter) { +func getFilterSets(terms []v1.SubnetSelectorTerm) (res [][]*ec2.Filter) { idFilter := &ec2.Filter{Name: aws.String("subnet-id")} for _, term := range terms { switch { diff --git a/pkg/providers/subnet/suite_test.go b/pkg/providers/subnet/suite_test.go index 08b7354f7924..f6dedfa1d2e7 100644 --- a/pkg/providers/subnet/suite_test.go +++ b/pkg/providers/subnet/suite_test.go @@ -16,32 +16,34 @@ package subnet_test import ( "context" + "sort" + "sync" "testing" - "github.com/aws/aws-sdk-go/aws" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/samber/lo" "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/test" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" ) var ctx context.Context var stop context.CancelFunc var env *coretest.Environment var awsEnv *test.Environment -var nodeClass *v1beta1.EC2NodeClass +var nodeClass *v1.EC2NodeClass func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -50,7 +52,7 @@ func TestAWS(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) ctx, stop = context.WithCancel(ctx) @@ -65,17 +67,19 @@ var _ = AfterSuite(func() { var _ = BeforeEach(func() { ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) - nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ - AMIFamily: aws.String(v1beta1.AMIFamilyAL2), - SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ + nodeClass = test.EC2NodeClass(v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + AMISelectorTerms: []v1.AMISelectorTerm{{ + Alias: "al2@latest", + }}, + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ { Tags: map[string]string{ "*": "*", }, }, }, - SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{ "*": "*", @@ -94,7 +98,7 @@ var _ = AfterEach(func() { var _ = Describe("SubnetProvider", func() { Context("List", func() { It("should discover subnet by ID", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { ID: "subnet-test1", }, @@ -105,12 +109,13 @@ var _ = Describe("SubnetProvider", func() { { SubnetId: lo.ToPtr("subnet-test1"), AvailabilityZone: lo.ToPtr("test-zone-1a"), + AvailabilityZoneId: lo.ToPtr("tstz1-1a"), AvailableIpAddressCount: lo.ToPtr[int64](100), }, }, subnets) }) It("should discover subnets by IDs", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { ID: "subnet-test1", }, @@ -124,17 +129,19 @@ var _ = Describe("SubnetProvider", func() { { SubnetId: lo.ToPtr("subnet-test1"), AvailabilityZone: lo.ToPtr("test-zone-1a"), + AvailabilityZoneId: lo.ToPtr("tstz1-1a"), AvailableIpAddressCount: lo.ToPtr[int64](100), }, { SubnetId: lo.ToPtr("subnet-test2"), AvailabilityZone: lo.ToPtr("test-zone-1b"), + AvailabilityZoneId: lo.ToPtr("tstz1-1b"), AvailableIpAddressCount: lo.ToPtr[int64](100), }, }, subnets) }) It("should discover subnets by IDs and tags", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { ID: "subnet-test1", Tags: map[string]string{"foo": "bar"}, @@ -150,17 +157,19 @@ var _ = Describe("SubnetProvider", func() { { SubnetId: lo.ToPtr("subnet-test1"), AvailabilityZone: lo.ToPtr("test-zone-1a"), + AvailabilityZoneId: lo.ToPtr("tstz1-1a"), AvailableIpAddressCount: lo.ToPtr[int64](100), }, { SubnetId: lo.ToPtr("subnet-test2"), AvailabilityZone: lo.ToPtr("test-zone-1b"), + AvailabilityZoneId: lo.ToPtr("tstz1-1b"), AvailableIpAddressCount: lo.ToPtr[int64](100), }, }, subnets) }) It("should discover subnets by a single tag", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { Tags: map[string]string{"Name": "test-subnet-1"}, }, @@ -171,12 +180,13 @@ var _ = Describe("SubnetProvider", func() { { SubnetId: lo.ToPtr("subnet-test1"), AvailabilityZone: lo.ToPtr("test-zone-1a"), + AvailabilityZoneId: lo.ToPtr("tstz1-1a"), AvailableIpAddressCount: lo.ToPtr[int64](100), }, }, subnets) }) It("should discover subnets by multiple tag values", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { Tags: map[string]string{"Name": "test-subnet-1"}, }, @@ -190,17 +200,19 @@ var _ = Describe("SubnetProvider", func() { { SubnetId: lo.ToPtr("subnet-test1"), AvailabilityZone: lo.ToPtr("test-zone-1a"), + AvailabilityZoneId: lo.ToPtr("tstz1-1a"), AvailableIpAddressCount: lo.ToPtr[int64](100), }, { SubnetId: lo.ToPtr("subnet-test2"), AvailabilityZone: lo.ToPtr("test-zone-1b"), + AvailabilityZoneId: lo.ToPtr("tstz1-1b"), AvailableIpAddressCount: lo.ToPtr[int64](100), }, }, subnets) }) It("should discover subnets by IDs intersected with tags", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { ID: "subnet-test2", Tags: map[string]string{"foo": "bar"}, @@ -212,39 +224,17 @@ var _ = Describe("SubnetProvider", func() { { SubnetId: lo.ToPtr("subnet-test2"), AvailabilityZone: lo.ToPtr("test-zone-1b"), + AvailabilityZoneId: lo.ToPtr("tstz1-1b"), AvailableIpAddressCount: lo.ToPtr[int64](100), }, }, subnets) }) }) - Context("CheckAnyPublicIPAssociations", func() { - It("should note that no subnets assign a public IPv4 address to EC2 instances on launch", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - ID: "subnet-test1", - Tags: map[string]string{"foo": "bar"}, - }, - } - onlyPrivate, err := awsEnv.SubnetProvider.CheckAnyPublicIPAssociations(ctx, nodeClass) - Expect(err).To(BeNil()) - Expect(onlyPrivate).To(BeFalse()) - }) - It("should note that at least one subnet assigns a public IPv4 address to EC2instances on launch", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - ID: "subnet-test2", - }, - } - onlyPrivate, err := awsEnv.SubnetProvider.CheckAnyPublicIPAssociations(ctx, nodeClass) - Expect(err).To(BeNil()) - Expect(onlyPrivate).To(BeTrue()) - }) - }) Context("Provider Cache", func() { It("should resolve subnets from cache that are filtered by id", func() { expectedSubnets := awsEnv.EC2API.DescribeSubnetsOutput.Clone().Subnets for _, subnet := range expectedSubnets { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { ID: *subnet.SubnetId, }, @@ -269,7 +259,7 @@ var _ = Describe("SubnetProvider", func() { return map[string]string{"Name": lo.FromPtr(tag.Value)} }) for _, tag := range tagSet { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { Tags: tag, }, @@ -286,6 +276,97 @@ var _ = Describe("SubnetProvider", func() { } }) }) + It("should not cause data races when calling List() simultaneously", func() { + wg := sync.WaitGroup{} + for i := 0; i < 10000; i++ { + wg.Add(1) + go func() { + defer wg.Done() + defer GinkgoRecover() + subnets, err := awsEnv.SubnetProvider.List(ctx, nodeClass) + Expect(err).ToNot(HaveOccurred()) + + Expect(subnets).To(HaveLen(4)) + // Sort everything in parallel and ensure that we don't get data races + sort.Slice(subnets, func(i, j int) bool { + if int(*subnets[i].AvailableIpAddressCount) != int(*subnets[j].AvailableIpAddressCount) { + return int(*subnets[i].AvailableIpAddressCount) > int(*subnets[j].AvailableIpAddressCount) + } + return *subnets[i].SubnetId < *subnets[j].SubnetId + }) + Expect(subnets).To(BeEquivalentTo([]*ec2.Subnet{ + { + AvailabilityZone: lo.ToPtr("test-zone-1a"), + AvailabilityZoneId: lo.ToPtr("tstz1-1a"), + AvailableIpAddressCount: lo.ToPtr[int64](100), + SubnetId: lo.ToPtr("subnet-test1"), + MapPublicIpOnLaunch: lo.ToPtr(false), + Tags: []*ec2.Tag{ + { + Key: lo.ToPtr("Name"), + Value: lo.ToPtr("test-subnet-1"), + }, + { + Key: lo.ToPtr("foo"), + Value: lo.ToPtr("bar"), + }, + }, + }, + { + AvailabilityZone: lo.ToPtr("test-zone-1b"), + AvailabilityZoneId: lo.ToPtr("tstz1-1b"), + AvailableIpAddressCount: lo.ToPtr[int64](100), + MapPublicIpOnLaunch: lo.ToPtr(true), + SubnetId: lo.ToPtr("subnet-test2"), + + Tags: []*ec2.Tag{ + { + Key: lo.ToPtr("Name"), + Value: lo.ToPtr("test-subnet-2"), + }, + { + Key: lo.ToPtr("foo"), + Value: lo.ToPtr("bar"), + }, + }, + }, + { + AvailabilityZone: lo.ToPtr("test-zone-1c"), + AvailabilityZoneId: lo.ToPtr("tstz1-1c"), + AvailableIpAddressCount: lo.ToPtr[int64](100), + SubnetId: lo.ToPtr("subnet-test3"), + Tags: []*ec2.Tag{ + { + Key: lo.ToPtr("Name"), + Value: lo.ToPtr("test-subnet-3"), + }, + { + Key: lo.ToPtr("TestTag"), + }, + { + Key: lo.ToPtr("foo"), + Value: lo.ToPtr("bar"), + }, + }, + }, + { + AvailabilityZone: lo.ToPtr("test-zone-1a-local"), + AvailabilityZoneId: lo.ToPtr("tstz1-1alocal"), + AvailableIpAddressCount: lo.ToPtr[int64](100), + SubnetId: lo.ToPtr("subnet-test4"), + MapPublicIpOnLaunch: lo.ToPtr(true), + Tags: []*ec2.Tag{ + { + Key: lo.ToPtr("Name"), + Value: lo.ToPtr("test-subnet-4"), + }, + }, + }, + })) + }() + } + wg.Wait() + }) }) func ExpectConsistsOfSubnets(expected, actual []*ec2.Subnet) { @@ -294,6 +375,7 @@ func ExpectConsistsOfSubnets(expected, actual []*ec2.Subnet) { for _, elem := range expected { _, ok := lo.Find(actual, func(s *ec2.Subnet) bool { return lo.FromPtr(s.SubnetId) == lo.FromPtr(elem.SubnetId) && + lo.FromPtr(s.AvailabilityZoneId) == lo.FromPtr(elem.AvailabilityZoneId) && lo.FromPtr(s.AvailabilityZone) == lo.FromPtr(elem.AvailabilityZone) && lo.FromPtr(s.AvailableIpAddressCount) == lo.FromPtr(elem.AvailableIpAddressCount) }) diff --git a/pkg/providers/version/version.go b/pkg/providers/version/version.go index c0c201cd7c7e..e021cd0615eb 100644 --- a/pkg/providers/version/version.go +++ b/pkg/providers/version/version.go @@ -17,12 +17,14 @@ package version import ( "context" "fmt" + "strconv" "strings" "github.com/patrickmn/go-cache" + "github.com/samber/lo" "k8s.io/apimachinery/pkg/util/version" "k8s.io/client-go/kubernetes" - "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/karpenter/pkg/utils/pretty" ) @@ -32,8 +34,8 @@ const ( // Karpenter's supported version of Kubernetes // If a user runs a karpenter image on a k8s version outside the min and max, // One error message will be fired to notify - MinK8sVersion = "1.23" - MaxK8sVersion = "1.29" + MinK8sVersion = "1.25" + MaxK8sVersion = "1.30" ) type Provider interface { @@ -67,14 +69,27 @@ func (p *DefaultProvider) Get(ctx context.Context) (string, error) { version := fmt.Sprintf("%s.%s", serverVersion.Major, strings.TrimSuffix(serverVersion.Minor, "+")) p.cache.SetDefault(kubernetesVersionCacheKey, version) if p.cm.HasChanged("kubernetes-version", version) { - logging.FromContext(ctx).With("version", version).Debugf("discovered kubernetes version") + log.FromContext(ctx).WithValues("version", version).V(1).Info("discovered kubernetes version") if err := validateK8sVersion(version); err != nil { - logging.FromContext(ctx).Error(err) + log.FromContext(ctx).Error(err, "failed validating kubernetes version") } } return version, nil } +// SupportedK8sVersions returns a slice of version strings in format "major.minor" for all versions of k8s supported by +// this version of Karpenter. +// Note: Assumes k8s only has a single major version (1.x) +func SupportedK8sVersions() []string { + minMinor := lo.Must(strconv.Atoi(strings.Split(MinK8sVersion, ".")[1])) + maxMinor := lo.Must(strconv.Atoi(strings.Split(MaxK8sVersion, ".")[1])) + versions := make([]string, 0, maxMinor-minMinor+1) + for i := minMinor; i <= maxMinor; i++ { + versions = append(versions, fmt.Sprintf("1.%d", i)) + } + return versions +} + func validateK8sVersion(v string) error { k8sVersion := version.MustParseGeneric(v) diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 0a0b1ca56e51..ac4c357bfee1 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -21,12 +21,10 @@ import ( "github.com/patrickmn/go-cache" "github.com/samber/lo" corev1 "k8s.io/api/core/v1" - "knative.dev/pkg/ptr" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "sigs.k8s.io/karpenter/pkg/operator/scheme" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "github.com/aws/karpenter-provider-aws/pkg/apis" awscache "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" @@ -36,6 +34,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" "github.com/aws/karpenter-provider-aws/pkg/providers/securitygroup" + ssmp "github.com/aws/karpenter-provider-aws/pkg/providers/ssm" "github.com/aws/karpenter-provider-aws/pkg/providers/subnet" "github.com/aws/karpenter-provider-aws/pkg/providers/version" @@ -45,8 +44,8 @@ import ( ) func init() { - lo.Must0(apis.AddToScheme(scheme.Scheme)) - corev1beta1.NormalizedLabels = lo.Assign(corev1beta1.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": corev1.LabelTopologyZone}) + karpv1beta1.NormalizedLabels = lo.Assign(karpv1.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": corev1.LabelTopologyZone}) + karpv1.NormalizedLabels = lo.Assign(karpv1.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": corev1.LabelTopologyZone}) } type Environment struct { @@ -58,14 +57,17 @@ type Environment struct { PricingAPI *fake.PricingAPI // Cache - EC2Cache *cache.Cache - KubernetesVersionCache *cache.Cache - InstanceTypeCache *cache.Cache - UnavailableOfferingsCache *awscache.UnavailableOfferings - LaunchTemplateCache *cache.Cache - SubnetCache *cache.Cache - SecurityGroupCache *cache.Cache - InstanceProfileCache *cache.Cache + EC2Cache *cache.Cache + KubernetesVersionCache *cache.Cache + InstanceTypeCache *cache.Cache + UnavailableOfferingsCache *awscache.UnavailableOfferings + LaunchTemplateCache *cache.Cache + SubnetCache *cache.Cache + AvailableIPAdressCache *cache.Cache + AssociatePublicIPAddressCache *cache.Cache + SecurityGroupCache *cache.Cache + InstanceProfileCache *cache.Cache + SSMCache *cache.Cache // Providers InstanceTypesProvider *instancetype.DefaultProvider @@ -94,17 +96,21 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment unavailableOfferingsCache := awscache.NewUnavailableOfferings() launchTemplateCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) subnetCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + availableIPAdressCache := cache.New(awscache.AvailableIPAddressTTL, awscache.DefaultCleanupInterval) + associatePublicIPAddressCache := cache.New(awscache.AssociatePublicIPAddressTTL, awscache.DefaultCleanupInterval) securityGroupCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) instanceProfileCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ssmCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) fakePricingAPI := &fake.PricingAPI{} // Providers pricingProvider := pricing.NewDefaultProvider(ctx, fakePricingAPI, ec2api, fake.DefaultRegion) - subnetProvider := subnet.NewDefaultProvider(ec2api, subnetCache) + subnetProvider := subnet.NewDefaultProvider(ec2api, subnetCache, availableIPAdressCache, associatePublicIPAddressCache) securityGroupProvider := securitygroup.NewDefaultProvider(ec2api, securityGroupCache) versionProvider := version.NewDefaultProvider(env.KubernetesInterface, kubernetesVersionCache) instanceProfileProvider := instanceprofile.NewDefaultProvider(fake.DefaultRegion, iamapi, instanceProfileCache) - amiProvider := amifamily.NewDefaultProvider(versionProvider, ssmapi, ec2api, ec2Cache) + ssmProvider := ssmp.NewDefaultProvider(ssmapi, ssmCache) + amiProvider := amifamily.NewDefaultProvider(versionProvider, ssmProvider, ec2api, ec2Cache) amiResolver := amifamily.NewResolver(amiProvider) instanceTypesProvider := instancetype.NewDefaultProvider(fake.DefaultRegion, instanceTypeCache, ec2api, subnetProvider, unavailableOfferingsCache, pricingProvider) launchTemplateProvider := @@ -116,7 +122,7 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment amiResolver, securityGroupProvider, subnetProvider, - ptr.String("ca-bundle"), + lo.ToPtr("ca-bundle"), make(chan struct{}), net.ParseIP("10.0.100.10"), "https://test-cluster", @@ -138,14 +144,16 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment IAMAPI: iamapi, PricingAPI: fakePricingAPI, - EC2Cache: ec2Cache, - KubernetesVersionCache: kubernetesVersionCache, - InstanceTypeCache: instanceTypeCache, - LaunchTemplateCache: launchTemplateCache, - SubnetCache: subnetCache, - SecurityGroupCache: securityGroupCache, - InstanceProfileCache: instanceProfileCache, - UnavailableOfferingsCache: unavailableOfferingsCache, + EC2Cache: ec2Cache, + KubernetesVersionCache: kubernetesVersionCache, + LaunchTemplateCache: launchTemplateCache, + SubnetCache: subnetCache, + AvailableIPAdressCache: availableIPAdressCache, + AssociatePublicIPAddressCache: associatePublicIPAddressCache, + SecurityGroupCache: securityGroupCache, + InstanceProfileCache: instanceProfileCache, + UnavailableOfferingsCache: unavailableOfferingsCache, + SSMCache: ssmCache, InstanceTypesProvider: instanceTypesProvider, InstanceProvider: instanceProvider, @@ -167,16 +175,18 @@ func (env *Environment) Reset() { env.IAMAPI.Reset() env.PricingAPI.Reset() env.PricingProvider.Reset() + env.InstanceTypesProvider.Reset() env.EC2Cache.Flush() env.KubernetesVersionCache.Flush() - env.InstanceTypeCache.Flush() env.UnavailableOfferingsCache.Flush() env.LaunchTemplateCache.Flush() env.SubnetCache.Flush() + env.AssociatePublicIPAddressCache.Flush() + env.AvailableIPAdressCache.Flush() env.SecurityGroupCache.Flush() env.InstanceProfileCache.Flush() - + env.SSMCache.Flush() mfs, err := crmetrics.Registry.Gather() if err != nil { for _, mf := range mfs { diff --git a/pkg/test/nodeclass.go b/pkg/test/nodeclass.go index 4aa58d4a75d3..ecba27c5c8c9 100644 --- a/pkg/test/nodeclass.go +++ b/pkg/test/nodeclass.go @@ -19,16 +19,118 @@ import ( "fmt" "github.com/imdario/mergo" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/test" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" ) -func EC2NodeClass(overrides ...v1beta1.EC2NodeClass) *v1beta1.EC2NodeClass { +func EC2NodeClass(overrides ...v1.EC2NodeClass) *v1.EC2NodeClass { + options := v1.EC2NodeClass{} + for _, override := range overrides { + if err := mergo.Merge(&options, override, mergo.WithOverride); err != nil { + panic(fmt.Sprintf("Failed to merge settings: %s", err)) + } + } + if len(options.Spec.AMISelectorTerms) == 0 { + options.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} + options.Status.AMIs = []v1.AMI{ + { + ID: "ami-test1", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureAmd64}}, + {Key: v1.LabelInstanceGPUCount, Operator: corev1.NodeSelectorOpDoesNotExist}, + {Key: v1.LabelInstanceAcceleratorCount, Operator: corev1.NodeSelectorOpDoesNotExist}, + }, + }, + { + ID: "ami-test2", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureAmd64}}, + {Key: v1.LabelInstanceGPUCount, Operator: corev1.NodeSelectorOpExists}, + }, + }, + { + ID: "ami-test3", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureAmd64}}, + {Key: v1.LabelInstanceAcceleratorCount, Operator: corev1.NodeSelectorOpExists}, + }, + }, + { + ID: "ami-test4", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureArm64}}, + {Key: v1.LabelInstanceGPUCount, Operator: corev1.NodeSelectorOpDoesNotExist}, + {Key: v1.LabelInstanceAcceleratorCount, Operator: corev1.NodeSelectorOpDoesNotExist}, + }, + }, + } + } + if options.Spec.Role == "" { + options.Spec.Role = "test-role" + options.Status.InstanceProfile = "test-profile" + } + if len(options.Spec.SecurityGroupSelectorTerms) == 0 { + options.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "*": "*", + }, + }, + } + options.Status.SecurityGroups = []v1.SecurityGroup{ + { + ID: "sg-test1", + }, + { + ID: "sg-test2", + }, + { + ID: "sg-test3", + }, + } + } + if len(options.Spec.SubnetSelectorTerms) == 0 { + options.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "*": "*", + }, + }, + } + options.Status.Subnets = []v1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + ZoneID: "tstz1-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + ZoneID: "tstz1-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + ZoneID: "tstz1-1c", + }, + } + } + return &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(options.ObjectMeta), + Spec: options.Spec, + Status: options.Status, + } +} + +func BetaEC2NodeClass(overrides ...v1beta1.EC2NodeClass) *v1beta1.EC2NodeClass { options := v1beta1.EC2NodeClass{} for _, override := range overrides { if err := mergo.Merge(&options, override, mergo.WithOverride); err != nil { @@ -37,6 +139,38 @@ func EC2NodeClass(overrides ...v1beta1.EC2NodeClass) *v1beta1.EC2NodeClass { } if options.Spec.AMIFamily == nil { options.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + options.Status.AMIs = []v1beta1.AMI{ + { + ID: "ami-test1", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1beta1.ArchitectureAmd64}}, + {Key: v1beta1.LabelInstanceGPUCount, Operator: corev1.NodeSelectorOpDoesNotExist}, + {Key: v1beta1.LabelInstanceAcceleratorCount, Operator: corev1.NodeSelectorOpDoesNotExist}, + }, + }, + { + ID: "ami-test2", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1beta1.ArchitectureAmd64}}, + {Key: v1beta1.LabelInstanceGPUCount, Operator: corev1.NodeSelectorOpExists}, + }, + }, + { + ID: "ami-test3", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1beta1.ArchitectureAmd64}}, + {Key: v1beta1.LabelInstanceAcceleratorCount, Operator: corev1.NodeSelectorOpExists}, + }, + }, + { + ID: "ami-test4", + Requirements: []corev1.NodeSelectorRequirement{ + {Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1beta1.ArchitectureArm64}}, + {Key: v1beta1.LabelInstanceGPUCount, Operator: corev1.NodeSelectorOpDoesNotExist}, + {Key: v1beta1.LabelInstanceAcceleratorCount, Operator: corev1.NodeSelectorOpDoesNotExist}, + }, + }, + } } if options.Spec.Role == "" { options.Spec.Role = "test-role" @@ -50,6 +184,17 @@ func EC2NodeClass(overrides ...v1beta1.EC2NodeClass) *v1beta1.EC2NodeClass { }, }, } + options.Status.SecurityGroups = []v1beta1.SecurityGroup{ + { + ID: "sg-test1", + }, + { + ID: "sg-test2", + }, + { + ID: "sg-test3", + }, + } } if len(options.Spec.SubnetSelectorTerms) == 0 { options.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ @@ -59,6 +204,23 @@ func EC2NodeClass(overrides ...v1beta1.EC2NodeClass) *v1beta1.EC2NodeClass { }, }, } + options.Status.Subnets = []v1beta1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + ZoneID: "tstz1-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + ZoneID: "tstz1-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + ZoneID: "tstz1-1c", + }, + } } return &v1beta1.EC2NodeClass{ ObjectMeta: test.ObjectMeta(options.ObjectMeta), @@ -69,8 +231,8 @@ func EC2NodeClass(overrides ...v1beta1.EC2NodeClass) *v1beta1.EC2NodeClass { func EC2NodeClassFieldIndexer(ctx context.Context) func(cache.Cache) error { return func(c cache.Cache) error { - return c.IndexField(ctx, &corev1beta1.NodeClaim{}, "spec.nodeClassRef.name", func(obj client.Object) []string { - nc := obj.(*corev1beta1.NodeClaim) + return c.IndexField(ctx, &karpv1.NodeClaim{}, "spec.nodeClassRef.name", func(obj client.Object) []string { + nc := obj.(*karpv1.NodeClaim) if nc.Spec.NodeClassRef == nil { return []string{""} } diff --git a/pkg/test/options.go b/pkg/test/options.go index 4657f28e9c53..12739f6986ee 100644 --- a/pkg/test/options.go +++ b/pkg/test/options.go @@ -16,7 +16,6 @@ package test import ( "fmt" - "time" "github.com/imdario/mergo" "github.com/samber/lo" @@ -25,8 +24,6 @@ import ( ) type OptionsFields struct { - AssumeRoleARN *string - AssumeRoleDuration *time.Duration ClusterCABundle *string ClusterName *string ClusterEndpoint *string @@ -44,8 +41,6 @@ func Options(overrides ...OptionsFields) *options.Options { } } return &options.Options{ - AssumeRoleARN: lo.FromPtrOr(opts.AssumeRoleARN, ""), - AssumeRoleDuration: lo.FromPtrOr(opts.AssumeRoleDuration, 15*time.Minute), ClusterCABundle: lo.FromPtrOr(opts.ClusterCABundle, ""), ClusterName: lo.FromPtrOr(opts.ClusterName, "test-cluster"), ClusterEndpoint: lo.FromPtrOr(opts.ClusterEndpoint, "https://test-cluster"), diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 2dc741e87e3d..fb7fab123d66 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -15,13 +15,24 @@ limitations under the License. package utils import ( + "context" + "encoding/json" "fmt" + "os" "regexp" + "strconv" "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/hashstructure/v2" "github.com/samber/lo" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" ) var ( @@ -66,3 +77,83 @@ func PrettySlice[T any](s []T, maxItems int) string { } return sb.String() } + +// WithDefaultFloat64 returns the float64 value of the supplied environment variable or, if not present, +// the supplied default value. If the float64 conversion fails, returns the default +func WithDefaultFloat64(key string, def float64) float64 { + val, ok := os.LookupEnv(key) + if !ok { + return def + } + f, err := strconv.ParseFloat(val, 64) + if err != nil { + return def + } + return f +} + +// GetKubletConfigurationWithNodePool use the most recent version of the kubelet configuration. +// The priority of fields is listed below: +// 1.) v1 NodePool kubelet annotation (Showing a user configured using v1beta1 NodePool at some point) +// 2.) v1 EC2NodeClass will be used (showing a user configured using v1 EC2NodeClass) +func GetKubletConfigurationWithNodePool(nodePool *karpv1.NodePool, nodeClass *v1.EC2NodeClass) (*v1.KubeletConfiguration, error) { + if nodePool != nil { + if annotation, ok := nodePool.Annotations[karpv1.KubeletCompatibilityAnnotationKey]; ok { + return parseKubeletConfiguration(annotation) + } + } + return nodeClass.Spec.Kubelet, nil +} + +func GetKubeletConfigurationWithNodeClaim(nodeClaim *karpv1.NodeClaim, nodeClass *v1.EC2NodeClass) (*v1.KubeletConfiguration, error) { + if annotation, ok := nodeClaim.Annotations[karpv1.KubeletCompatibilityAnnotationKey]; ok { + return parseKubeletConfiguration(annotation) + } + return nodeClass.Spec.Kubelet, nil +} + +func parseKubeletConfiguration(annotation string) (*v1.KubeletConfiguration, error) { + kubelet := &karpv1beta1.KubeletConfiguration{} + err := json.Unmarshal([]byte(annotation), kubelet) + if err != nil { + return nil, fmt.Errorf("parsing kubelet config from %s annotation, %w", karpv1.KubeletCompatibilityAnnotationKey, err) + } + return &v1.KubeletConfiguration{ + ClusterDNS: kubelet.ClusterDNS, + MaxPods: kubelet.MaxPods, + PodsPerCore: kubelet.PodsPerCore, + SystemReserved: kubelet.SystemReserved, + KubeReserved: kubelet.KubeReserved, + EvictionSoft: kubelet.EvictionSoft, + EvictionHard: kubelet.EvictionHard, + EvictionSoftGracePeriod: kubelet.EvictionSoftGracePeriod, + EvictionMaxPodGracePeriod: kubelet.EvictionMaxPodGracePeriod, + ImageGCHighThresholdPercent: kubelet.ImageGCHighThresholdPercent, + ImageGCLowThresholdPercent: kubelet.ImageGCLowThresholdPercent, + CPUCFSQuota: kubelet.CPUCFSQuota, + }, nil +} + +func GetHashKubelet(nodePool *karpv1.NodePool, nodeClass *v1.EC2NodeClass) (string, error) { + kubelet, err := GetKubletConfigurationWithNodePool(nodePool, nodeClass) + if err != nil { + return "", err + } + return fmt.Sprint(lo.Must(hashstructure.Hash(kubelet, hashstructure.FormatV2, &hashstructure.HashOptions{ + SlicesAsSets: true, + IgnoreZeroValue: true, + ZeroNil: true, + }))), nil +} + +func ResolveNodePoolFromNodeClaim(ctx context.Context, kubeClient client.Client, nodeClaim *karpv1.NodeClaim) (*karpv1.NodePool, error) { + if nodePoolName, ok := nodeClaim.Labels[karpv1.NodePoolLabelKey]; ok { + nodePool := &karpv1.NodePool{} + if err := kubeClient.Get(ctx, types.NamespacedName{Name: nodePoolName}, nodePool); err != nil { + return nil, err + } + return nodePool, nil + } + // There will be no nodePool referenced inside the nodeClaim in case of standalone nodeClaims + return nil, nil +} diff --git a/pkg/webhooks/webhooks.go b/pkg/webhooks/webhooks.go index 8c81ba72b6a4..95b08b2e8581 100644 --- a/pkg/webhooks/webhooks.go +++ b/pkg/webhooks/webhooks.go @@ -21,40 +21,37 @@ import ( "knative.dev/pkg/configmap" "knative.dev/pkg/controller" knativeinjection "knative.dev/pkg/injection" - "knative.dev/pkg/webhook/resourcesemantics" - "knative.dev/pkg/webhook/resourcesemantics/defaulting" - "knative.dev/pkg/webhook/resourcesemantics/validation" + "knative.dev/pkg/webhook/resourcesemantics/conversion" + "github.com/awslabs/operatorpkg/object" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" ) +var ( + ConversionResource = map[schema.GroupKind]conversion.GroupKindConversion{ + object.GVK(&v1.EC2NodeClass{}).GroupKind(): { + DefinitionName: "ec2nodeclasses.karpenter.k8s.aws", + HubVersion: "v1", + Zygotes: map[string]conversion.ConvertibleObject{ + "v1": &v1.EC2NodeClass{}, + "v1beta1": &v1beta1.EC2NodeClass{}, + }, + }, + } +) + func NewWebhooks() []knativeinjection.ControllerConstructor { return []knativeinjection.ControllerConstructor{ - NewCRDDefaultingWebhook, - NewCRDValidationWebhook, + NewCRDConversionWebhook, } } -func NewCRDDefaultingWebhook(ctx context.Context, _ configmap.Watcher) *controller.Impl { - return defaulting.NewAdmissionController(ctx, - "defaulting.webhook.karpenter.k8s.aws", - "/default/karpenter.k8s.aws", - Resources, - func(ctx context.Context) context.Context { return ctx }, - true, - ) -} - -func NewCRDValidationWebhook(ctx context.Context, _ configmap.Watcher) *controller.Impl { - return validation.NewAdmissionController(ctx, - "validation.webhook.karpenter.k8s.aws", - "/validate/karpenter.k8s.aws", - Resources, +func NewCRDConversionWebhook(ctx context.Context, _ configmap.Watcher) *controller.Impl { + return conversion.NewConversionController(ctx, + "/conversion/karpenter.k8s.aws", + ConversionResource, func(ctx context.Context) context.Context { return ctx }, - true, ) } - -var Resources = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{ - v1beta1.SchemeGroupVersion.WithKind("EC2NodeClass"): &v1beta1.EC2NodeClass{}, -} diff --git a/test/hack/e2e_scripts/diff_karpenter.sh b/test/hack/e2e_scripts/diff_karpenter.sh index 42d6e5274a0f..185b3288a962 100755 --- a/test/hack/e2e_scripts/diff_karpenter.sh +++ b/test/hack/e2e_scripts/diff_karpenter.sh @@ -6,4 +6,4 @@ fi helm diff upgrade --namespace kube-system \ karpenter "${CHART}" \ --version 0-$(git rev-parse HEAD) \ ---reuse-values --three-way-merge --detailed-exitcode +--reuse-values --three-way-merge --detailed-exitcode --no-hooks diff --git a/test/hack/e2e_scripts/install_karpenter.sh b/test/hack/e2e_scripts/install_karpenter.sh index dbe1aba20d71..1f1f99279efc 100755 --- a/test/hack/e2e_scripts/install_karpenter.sh +++ b/test/hack/e2e_scripts/install_karpenter.sh @@ -1,20 +1,14 @@ aws eks update-kubeconfig --name "$CLUSTER_NAME" -# Parse minor version to determine whether to enable the webhooks -K8S_VERSION_MINOR="${K8S_VERSION#*.}" -WEBHOOK_ENABLED=false -if (( K8S_VERSION_MINOR < 25 )); then - WEBHOOK_ENABLED=true -fi +WEBHOOK_ENABLED=true CHART="oci://$ECR_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/karpenter/snapshot/karpenter" ADDITIONAL_FLAGS="" if (( "$PRIVATE_CLUSTER" == 'true' )); then CHART="oci://$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/karpenter/snapshot/karpenter" - ADDITIONAL_FLAGS="--set .Values.controller.image.repository=$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/karpenter/snapshot/controller --set .Values.controller.image.digest=\"\"" + ADDITIONAL_FLAGS="--set .Values.controller.image.repository=$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/karpenter/snapshot/controller --set .Values.controller.image.digest=\"\" --set .Values.postInstallHook.image.repository=$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/ecr-public/bitnami/kubectl --set .Values.postInstallHook.image.digest=\"\"" fi -# Remove service account annotation when dropping support for 1.23 helm upgrade --install karpenter "${CHART}" \ -n kube-system \ --version "0-$(git rev-parse HEAD)" \ @@ -26,9 +20,9 @@ helm upgrade --install karpenter "${CHART}" \ --set settings.clusterName="$CLUSTER_NAME" \ --set settings.interruptionQueue="$CLUSTER_NAME" \ --set settings.featureGates.spotToSpotConsolidation=true \ - --set controller.resources.requests.cpu=3 \ + --set controller.resources.requests.cpu=5 \ --set controller.resources.requests.memory=3Gi \ - --set controller.resources.limits.cpu=3 \ + --set controller.resources.limits.cpu=5 \ --set controller.resources.limits.memory=3Gi \ --set serviceMonitor.enabled=true \ --set serviceMonitor.additionalLabels.scrape=enabled \ diff --git a/test/hack/e2e_scripts/install_prometheus.sh b/test/hack/e2e_scripts/install_prometheus.sh index d05e463dd223..3907733b2882 100755 --- a/test/hack/e2e_scripts/install_prometheus.sh +++ b/test/hack/e2e_scripts/install_prometheus.sh @@ -1,5 +1,3 @@ -# Remove service account annotation when dropping support for 1.23 - CHART=prometheus-community/kube-prometheus-stack VALUES=./.github/actions/e2e/install-prometheus/values.yaml ENABLED=true diff --git a/test/hack/resource/clean/main.go b/test/hack/resource/clean/main.go index aed3d68d433e..dd8b1aa9ca46 100644 --- a/test/hack/resource/clean/main.go +++ b/test/hack/resource/clean/main.go @@ -34,9 +34,7 @@ import ( const sweeperCleanedResourcesTableName = "sweeperCleanedResources" -var excludedClusters = []string{ - "soak-periodic-2785632730", -} +var excludedClusters = []string{} func main() { expiration := flag.String("expiration", "12h", "define the expirationTTL of the resources") @@ -45,6 +43,7 @@ func main() { ctx := context.Background() cfg := lo.Must(config.LoadDefaultConfig(ctx)) + cfg.RetryMaxAttempts = 10 logger := lo.Must(zap.NewProduction()).Sugar() diff --git a/test/hack/resource/go.mod b/test/hack/resource/go.mod index af0f422f889a..fe3daf63140e 100644 --- a/test/hack/resource/go.mod +++ b/test/hack/resource/go.mod @@ -1,48 +1,48 @@ module github.com/aws/karpenter-provider-aws/test/hack/resource -go 1.22 +go 1.22.3 require ( - github.com/aws/aws-sdk-go-v2 v1.22.1 - github.com/aws/aws-sdk-go-v2/config v1.18.27 - github.com/aws/aws-sdk-go-v2/service/cloudformation v1.30.0 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0 - github.com/aws/aws-sdk-go-v2/service/iam v1.21.0 - github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.18.2 - github.com/samber/lo v1.38.1 + github.com/aws/aws-sdk-go-v2 v1.26.1 + github.com/aws/aws-sdk-go-v2/config v1.27.11 + github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 + github.com/aws/aws-sdk-go-v2/service/iam v1.32.0 + github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.25.5 + github.com/samber/lo v1.39.0 go.uber.org/multierr v1.11.0 - go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 - k8s.io/api v0.29.2 + go.uber.org/zap v1.27.0 + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f + k8s.io/api v0.30.0 ) require ( - github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.32 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect - github.com/aws/smithy-go v1.16.0 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect + github.com/aws/smithy-go v1.20.2 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - go.uber.org/atomic v1.7.0 // indirect - golang.org/x/net v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/text v0.15.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/apimachinery v0.29.2 // indirect - k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + k8s.io/apimachinery v0.30.0 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/test/hack/resource/go.sum b/test/hack/resource/go.sum index 3bdbdcc58303..ab9878db89cb 100644 --- a/test/hack/resource/go.sum +++ b/test/hack/resource/go.sum @@ -1,55 +1,46 @@ -github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.20.1/go.mod h1:NU06lETsFm8fUC6ZjhgDpVBcGZTFQ6XM+LZWZxMI4ac= -github.com/aws/aws-sdk-go-v2 v1.22.1 h1:sjnni/AuoTXxHitsIdT0FwmqUuNUuHtufcVDErVFT9U= -github.com/aws/aws-sdk-go-v2 v1.22.1/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c= -github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA= -github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw= -github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk= -github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38/go.mod h1:qggunOChCMu9ZF/UkAfhTz25+U2rLVb3ya0Ua6TTfCA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1 h1:fi1ga6WysOyYb5PAf3Exd6B5GiSNpnZim4h1rhlBqx0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1/go.mod h1:V5CY8wNurvPUibTi9mwqUqpiFZ5LnioKWIFUDtIzdI8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32/go.mod h1:0ZXSqrty4FtQ7p8TEuRde/SZm9X05KT18LAUlR40Ln0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1 h1:ZpaV/j48RlPc4AmOZuPv22pJliXjXq8/reL63YzyFnw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1/go.mod h1:R8aXraabD2e3qv1csxM14/X9WF4wFMIY0kH4YEtYD5M= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0= -github.com/aws/aws-sdk-go-v2/service/cloudformation v1.30.0 h1:XbDkc4FLeg1RfnqeblfbJvaEabqq9ByZl4zqyPFkfSc= -github.com/aws/aws-sdk-go-v2/service/cloudformation v1.30.0/go.mod h1:SwQFcCs9Rog8hSHm+81KBkAK+UKLXErA/1ChaEI8mLE= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0 h1:P4dyjm49F2kKws0FpouBC6fjVImACXKt752+CWa01lM= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0/go.mod h1:tIctCeX9IbzsUTKHt53SVEcgyfxV2ElxJeEB+QUbc4M= -github.com/aws/aws-sdk-go-v2/service/iam v1.21.0 h1:8hEpu60CWlrp7iEBUFRZhgPoX6+gadaGL1sD4LoRYS0= -github.com/aws/aws-sdk-go-v2/service/iam v1.21.0/go.mod h1:aQZ8BI+reeaY7RI/QQp7TKCSUHOesTdrzzylp3CW85c= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.32 h1:ltFklFRb78MNetqtmqZ/6Tc6i76QRMXxDe0LXYl/jd8= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.32/go.mod h1:jBlPRKTAedLFuhO71Wm5dgN9x+/pJ6TtwfQmq7RLvNk= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg= -github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.18.2 h1:5QyvAYyr+ZibpVxfovzd5JMTZ8miv9s3zT4jG4PJkIA= -github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.18.2/go.mod h1:3ZCiyyNF7myh/a7DcOjcqRsLmSF9EdhEZSr00Qlui4s= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aws/smithy-go v1.14.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik= -github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= +github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= +github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= +github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0 h1:Ap5tOJfeAH1hO2UQc3X3uMlwP7uryFeZXMvZCXIlLSE= +github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0/go.mod h1:/v2KYdCW4BaHKayenaWEXOOdxItIwEA3oU0XzuQY3F0= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 h1:ooy0OFbrdSwgk32OFGPnvBwry5ySYCKkgTEbQ2hejs8= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0/go.mod h1:xejKuuRDjz6z5OqyeLsz01MlOqqW7CqpAB4PabNvpu8= +github.com/aws/aws-sdk-go-v2/service/iam v1.32.0 h1:ZNlfPdw849gBo/lvLFbEEvpTJMij0LXqiNWZ+lIamlU= +github.com/aws/aws-sdk-go-v2/service/iam v1.32.0/go.mod h1:aXWImQV0uTW35LM0A/T4wEg6R1/ReXUu4SM6/lUHYK0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.6 h1:6tayEze2Y+hiL3kdnEUxSPsP+pJsUfwLSFspFl1ru9Q= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.6/go.mod h1:qVNb/9IOVsLCZh0x2lnagrBwQ9fxajUpXS7OZfIsKn0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= +github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.25.5 h1:0Ty3j3QkLoqkZ+VagFisIsKYxGAzjv9hIQb84nlt/Jc= +github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.25.5/go.mod h1:9R1IlrgiivwTCZdbKgMPkseFS+moUM+DLh0TEjO6pvE= +github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= +github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -73,14 +64,12 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= -github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= +github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -89,27 +78,25 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM= -golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -118,8 +105,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -138,14 +125,14 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= -k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= -k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= -k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA= +k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE= +k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA= +k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/test/hack/resource/pkg/resourcetypes/eni.go b/test/hack/resource/pkg/resourcetypes/eni.go index ac759d2d8db7..0f424a483b50 100644 --- a/test/hack/resource/pkg/resourcetypes/eni.go +++ b/test/hack/resource/pkg/resourcetypes/eni.go @@ -43,95 +43,67 @@ func (e *ENI) Global() bool { } func (e *ENI) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) { - var nextToken *string - for { - out, err := e.ec2Client.DescribeNetworkInterfaces(ctx, &ec2.DescribeNetworkInterfacesInput{ - Filters: []ec2types.Filter{ - { - Name: lo.ToPtr("tag-key"), - Values: []string{k8sClusterTag}, - }, + enis, err := e.getAllENIs(ctx, &ec2.DescribeNetworkInterfacesInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("tag-key"), + Values: []string{k8sClusterTag}, }, - NextToken: nextToken, + }, + }) + if err != nil { + return ids, err + } + + for _, ni := range enis { + clusterName, found := lo.Find(ni.TagSet, func(tag ec2types.Tag) bool { + return *tag.Key == k8sClusterTag }) - if err != nil { - return ids, err + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue } - - for _, ni := range out.NetworkInterfaces { - clusterName, found := lo.Find(ni.TagSet, func(tag ec2types.Tag) bool { - return *tag.Key == k8sClusterTag - }) - if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { - continue - } - creationDate, found := lo.Find(ni.TagSet, func(tag ec2types.Tag) bool { - return *tag.Key == "node.k8s.amazonaws.com/createdAt" - }) - if !found { - continue - } - creationTime, err := time.Parse(time.RFC3339, *creationDate.Value) - if err != nil { - continue - } - if ni.Status == ec2types.NetworkInterfaceStatusAvailable && creationTime.Before(expirationTime) { - ids = append(ids, lo.FromPtr(ni.NetworkInterfaceId)) - } + creationDate, found := lo.Find(ni.TagSet, func(tag ec2types.Tag) bool { + return *tag.Key == "node.k8s.amazonaws.com/createdAt" + }) + if !found { + continue } - - nextToken = out.NextToken - if nextToken == nil { - break + creationTime, err := time.Parse(time.RFC3339, *creationDate.Value) + if err != nil { + continue + } + if ni.Status == ec2types.NetworkInterfaceStatusAvailable && creationTime.Before(expirationTime) { + ids = append(ids, lo.FromPtr(ni.NetworkInterfaceId)) } } + return ids, err } func (e *ENI) CountAll(ctx context.Context) (count int, err error) { - var nextToken *string - for { - out, err := e.ec2Client.DescribeNetworkInterfaces(ctx, &ec2.DescribeNetworkInterfacesInput{ - NextToken: nextToken, - }) - if err != nil { - return count, err - } - - count += len(out.NetworkInterfaces) - - nextToken = out.NextToken - if nextToken == nil { - break - } + enis, err := e.getAllENIs(ctx, &ec2.DescribeNetworkInterfacesInput{}) + if err != nil { + return 0, err } - return count, err + + return len(enis), err } func (e *ENI) Get(ctx context.Context, clusterName string) (ids []string, err error) { - var nextToken *string - for { - out, err := e.ec2Client.DescribeNetworkInterfaces(ctx, &ec2.DescribeNetworkInterfacesInput{ - Filters: []ec2types.Filter{ - { - Name: lo.ToPtr("tag:" + k8sClusterTag), - Values: []string{clusterName}, - }, + enis, err := e.getAllENIs(ctx, &ec2.DescribeNetworkInterfacesInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("tag:" + k8sClusterTag), + Values: []string{clusterName}, }, - NextToken: nextToken, - }) - if err != nil { - return ids, err - } - - for _, ni := range out.NetworkInterfaces { - ids = append(ids, lo.FromPtr(ni.NetworkInterfaceId)) - } + }, + }) + if err != nil { + return ids, err + } - nextToken = out.NextToken - if nextToken == nil { - break - } + for _, ni := range enis { + ids = append(ids, lo.FromPtr(ni.NetworkInterfaceId)) } return ids, err } @@ -154,3 +126,17 @@ func (e *ENI) Cleanup(ctx context.Context, ids []string) ([]string, error) { return deleted, errs } + +func (e *ENI) getAllENIs(ctx context.Context, params *ec2.DescribeNetworkInterfacesInput) (enis []ec2types.NetworkInterface, err error) { + paginator := ec2.NewDescribeNetworkInterfacesPaginator(e.ec2Client, params) + + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return enis, err + } + enis = append(enis, page.NetworkInterfaces...) + } + + return enis, nil +} diff --git a/test/hack/resource/pkg/resourcetypes/instance.go b/test/hack/resource/pkg/resourcetypes/instance.go index 089bda1b8d59..6c24ffa2ff10 100644 --- a/test/hack/resource/pkg/resourcetypes/instance.go +++ b/test/hack/resource/pkg/resourcetypes/instance.go @@ -21,6 +21,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/samber/lo" + "go.uber.org/multierr" "golang.org/x/exp/slices" ) @@ -41,123 +42,115 @@ func (i *Instance) Global() bool { } func (i *Instance) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) { - var nextToken *string - for { - out, err := i.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ - Filters: []ec2types.Filter{ - { - Name: lo.ToPtr("instance-state-name"), - Values: []string{string(ec2types.InstanceStateNameRunning)}, - }, - { - Name: lo.ToPtr("tag-key"), - Values: []string{karpenterNodePoolTag}, - }, + instances, err := i.getAllInstances(ctx, &ec2.DescribeInstancesInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("instance-state-name"), + Values: []string{string(ec2types.InstanceStateNameRunning)}, }, - NextToken: nextToken, - }) - if err != nil { - return ids, err - } + { + Name: lo.ToPtr("tag-key"), + Values: []string{karpenterNodePoolTag}, + }, + }, + }) + if err != nil { + return ids, err + } - for _, res := range out.Reservations { - for _, instance := range res.Instances { - clusterName, found := lo.Find(instance.Tags, func(tag ec2types.Tag) bool { - return *tag.Key == karpenterTestingTag - }) - if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { - continue - } - if lo.FromPtr(instance.LaunchTime).Before(expirationTime) { - ids = append(ids, lo.FromPtr(instance.InstanceId)) - } - } + for _, instance := range instances { + clusterName, found := lo.Find(instance.Tags, func(tag ec2types.Tag) bool { + return *tag.Key == karpenterTestingTag + }) + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue } - - nextToken = out.NextToken - if nextToken == nil { - break + if lo.FromPtr(instance.LaunchTime).Before(expirationTime) { + ids = append(ids, lo.FromPtr(instance.InstanceId)) } } + return ids, err } func (i *Instance) CountAll(ctx context.Context) (count int, err error) { - var nextToken *string - - for { - out, err := i.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ - Filters: []ec2types.Filter{ - { - Name: lo.ToPtr("instance-state-name"), - Values: []string{ - string(ec2types.InstanceStateNameRunning), - string(ec2types.InstanceStateNamePending), - string(ec2types.InstanceStateNameShuttingDown), - string(ec2types.InstanceStateNameStopped), - string(ec2types.InstanceStateNameStopping), - }, + instances, err := i.getAllInstances(ctx, &ec2.DescribeInstancesInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("instance-state-name"), + Values: []string{ + string(ec2types.InstanceStateNameRunning), + string(ec2types.InstanceStateNamePending), + string(ec2types.InstanceStateNameShuttingDown), + string(ec2types.InstanceStateNameStopped), + string(ec2types.InstanceStateNameStopping), }, }, - NextToken: nextToken, - }) - if err != nil { - return count, err - } - - for _, res := range out.Reservations { - count += len(res.Instances) - } - - nextToken = out.NextToken - if nextToken == nil { - break - } + }, + }) + if err != nil { + return count, err } - return count, err + + return len(instances), err } func (i *Instance) Get(ctx context.Context, clusterName string) (ids []string, err error) { - var nextToken *string - - for { - out, err := i.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ - Filters: []ec2types.Filter{ - { - Name: lo.ToPtr("instance-state-name"), - Values: []string{string(ec2types.InstanceStateNameRunning)}, - }, - { - Name: lo.ToPtr("tag:" + karpenterClusterNameTag), - Values: []string{clusterName}, - }, + instances, err := i.getAllInstances(ctx, &ec2.DescribeInstancesInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("instance-state-name"), + Values: []string{string(ec2types.InstanceStateNameRunning)}, }, - NextToken: nextToken, - }) - if err != nil { - return ids, err - } - - for _, res := range out.Reservations { - for _, instance := range res.Instances { - ids = append(ids, lo.FromPtr(instance.InstanceId)) - } - } + { + Name: lo.ToPtr("tag:" + karpenterClusterNameTag), + Values: []string{clusterName}, + }, + }, + }) + if err != nil { + return ids, err + } - nextToken = out.NextToken - if nextToken == nil { - break - } + for _, instance := range instances { + ids = append(ids, lo.FromPtr(instance.InstanceId)) } + return ids, err } // Cleanup any old instances that were managed by Karpenter or were provisioned as part of testing func (i *Instance) Cleanup(ctx context.Context, ids []string) ([]string, error) { - if _, err := i.ec2Client.TerminateInstances(ctx, &ec2.TerminateInstancesInput{ - InstanceIds: ids, - }); err != nil { - return nil, err + // The maximum number of EC2 instances which can be specified in a single API call + const maxIDCount = 1000 + chunkedIDs := lo.Chunk(ids, maxIDCount) + cleaned := make([]string, 0, len(ids)) + errs := make([]error, 0, len(chunkedIDs)) + for _, ids := range chunkedIDs { + if _, err := i.ec2Client.TerminateInstances(ctx, &ec2.TerminateInstancesInput{ + InstanceIds: ids, + }); err != nil { + errs = append(errs, err) + continue + } + cleaned = append(cleaned, ids...) } - return ids, nil + return cleaned, multierr.Combine(errs...) +} + +func (i *Instance) getAllInstances(ctx context.Context, params *ec2.DescribeInstancesInput) (instances []ec2types.Instance, err error) { + paginator := ec2.NewDescribeInstancesPaginator(i.ec2Client, params) + + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return instances, err + } + + for _, res := range page.Reservations { + instances = append(instances, res.Instances...) + } + } + + return instances, nil } diff --git a/test/hack/resource/pkg/resourcetypes/instanceprofile.go b/test/hack/resource/pkg/resourcetypes/instanceprofile.go index fa205f7ccc32..4b6b6edb96f1 100644 --- a/test/hack/resource/pkg/resourcetypes/instanceprofile.go +++ b/test/hack/resource/pkg/resourcetypes/instanceprofile.go @@ -44,15 +44,15 @@ func (ip *InstanceProfile) Global() bool { } func (ip *InstanceProfile) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (names []string, err error) { - out, err := ip.iamClient.ListInstanceProfiles(ctx, &iam.ListInstanceProfilesInput{}) + instanceProfiles, err := ip.getAllInstanceProfiles(ctx) if err != nil { return names, err } - errs := make([]error, len(out.InstanceProfiles)) - for i := range out.InstanceProfiles { + errs := make([]error, len(instanceProfiles)) + for i := range instanceProfiles { profiles, err := ip.iamClient.ListInstanceProfileTags(ctx, &iam.ListInstanceProfileTagsInput{ - InstanceProfileName: out.InstanceProfiles[i].InstanceProfileName, + InstanceProfileName: instanceProfiles[i].InstanceProfileName, }) if err != nil { errs[i] = err @@ -64,20 +64,16 @@ func (ip *InstanceProfile) GetExpired(ctx context.Context, expirationTime time.T }) // Checking to make sure we are only list resources in the given region region, _ := lo.Find(profiles.Tags, func(tag iamtypes.Tag) bool { - return lo.FromPtr(tag.Key) == v1.LabelTopologyZone + return lo.FromPtr(tag.Key) == v1.LabelTopologyRegion }) - if slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) || lo.FromPtr(region.Value) != lo.Must(config.LoadDefaultConfig(ctx)).Region { continue } - - for _, t := range profiles.Tags { - // Since we can only get the date of the instance profile (not the exact time the instance profile was created) - // we add a day to the time that it was created to account for the worst-case of the instance profile being created - // at 23:59:59 and being marked with a time of 00:00:00 due to only capturing the date and not the time - if lo.FromPtr(t.Key) == karpenterTestingTag && out.InstanceProfiles[i].CreateDate.Add(time.Hour*24).Before(expirationTime) { - names = append(names, lo.FromPtr(out.InstanceProfiles[i].InstanceProfileName)) - } + // Since we can only get the date of the instance profile (not the exact time the instance profile was created) + // we add a day to the time that it was created to account for the worst-case of the instance profile being created + // at 23:59:59 and being marked with a time of 00:00:00 due to only capturing the date and not the time + if lo.FromPtr(clusterName.Value) != "" && instanceProfiles[i].CreateDate.Add(time.Hour*24).Before(expirationTime) { + names = append(names, lo.FromPtr(instanceProfiles[i].InstanceProfileName)) } } @@ -85,24 +81,23 @@ func (ip *InstanceProfile) GetExpired(ctx context.Context, expirationTime time.T } func (ip *InstanceProfile) CountAll(ctx context.Context) (count int, err error) { - out, err := ip.iamClient.ListInstanceProfiles(ctx, &iam.ListInstanceProfilesInput{}) + instanceProfiles, err := ip.getAllInstanceProfiles(ctx) if err != nil { return count, err } - - return len(out.InstanceProfiles), nil + return len(instanceProfiles), err } func (ip *InstanceProfile) Get(ctx context.Context, clusterName string) (names []string, err error) { - out, err := ip.iamClient.ListInstanceProfiles(ctx, &iam.ListInstanceProfilesInput{}) + instanceProfiles, err := ip.getAllInstanceProfiles(ctx) if err != nil { return names, err } - errs := make([]error, len(out.InstanceProfiles)) - for i := range out.InstanceProfiles { + errs := make([]error, len(instanceProfiles)) + for i := range instanceProfiles { profiles, err := ip.iamClient.ListInstanceProfileTags(ctx, &iam.ListInstanceProfileTagsInput{ - InstanceProfileName: out.InstanceProfiles[i].InstanceProfileName, + InstanceProfileName: instanceProfiles[i].InstanceProfileName, }) if err != nil { errs[i] = err @@ -111,7 +106,7 @@ func (ip *InstanceProfile) Get(ctx context.Context, clusterName string) (names [ for _, t := range profiles.Tags { if lo.FromPtr(t.Key) == karpenterTestingTag && lo.FromPtr(t.Value) == clusterName { - names = append(names, lo.FromPtr(out.InstanceProfiles[i].InstanceProfileName)) + names = append(names, lo.FromPtr(instanceProfiles[i].InstanceProfileName)) } } } @@ -142,3 +137,17 @@ func (ip *InstanceProfile) Cleanup(ctx context.Context, names []string) ([]strin } return deleted, errs } + +func (ip *InstanceProfile) getAllInstanceProfiles(ctx context.Context) (instanceprofiles []iamtypes.InstanceProfile, err error) { + paginator := iam.NewListInstanceProfilesPaginator(ip.iamClient, &iam.ListInstanceProfilesInput{}) + + for paginator.HasMorePages() { + out, err := paginator.NextPage(ctx) + if err != nil { + return instanceprofiles, err + } + instanceprofiles = append(instanceprofiles, out.InstanceProfiles...) + } + + return instanceprofiles, nil +} diff --git a/test/hack/resource/pkg/resourcetypes/launchtemplate.go b/test/hack/resource/pkg/resourcetypes/launchtemplate.go index 18cfaccbf46f..9f4c0797abdd 100644 --- a/test/hack/resource/pkg/resourcetypes/launchtemplate.go +++ b/test/hack/resource/pkg/resourcetypes/launchtemplate.go @@ -42,86 +42,59 @@ func (lt *LaunchTemplate) Global() bool { } func (lt *LaunchTemplate) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (names []string, err error) { - var nextToken *string - for { - out, err := lt.ec2Client.DescribeLaunchTemplates(ctx, &ec2.DescribeLaunchTemplatesInput{ - Filters: []ec2types.Filter{ - { - Name: lo.ToPtr("tag-key"), - Values: []string{karpenterLaunchTemplateTag}, - }, + lts, err := lt.getAllLaunchTemplates(ctx, &ec2.DescribeLaunchTemplatesInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("tag-key"), + Values: []string{karpenterLaunchTemplateTag}, }, - NextToken: nextToken, - }) - if err != nil { - return names, err - } + }, + }) + if err != nil { + return names, err + } - for _, launchTemplate := range out.LaunchTemplates { - clusterName, found := lo.Find(launchTemplate.Tags, func(tag ec2types.Tag) bool { - return *tag.Key == k8sClusterTag - }) - if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { - continue - } - if lo.FromPtr(launchTemplate.CreateTime).Before(expirationTime) { - names = append(names, lo.FromPtr(launchTemplate.LaunchTemplateName)) - } + for _, launchtemplate := range lts { + clusterName, found := lo.Find(launchtemplate.Tags, func(tag ec2types.Tag) bool { + return *tag.Key == k8sClusterTag + }) + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue } - - nextToken = out.NextToken - if nextToken == nil { - break + if lo.FromPtr(launchtemplate.CreateTime).Before(expirationTime) { + names = append(names, lo.FromPtr(launchtemplate.LaunchTemplateName)) } } + return names, err } func (lt *LaunchTemplate) CountAll(ctx context.Context) (count int, err error) { - var nextToken *string - for { - out, err := lt.ec2Client.DescribeLaunchTemplates(ctx, &ec2.DescribeLaunchTemplatesInput{ - NextToken: nextToken, - }) - if err != nil { - return count, err - } - - count += len(out.LaunchTemplates) - - nextToken = out.NextToken - if nextToken == nil { - break - } + lts, err := lt.getAllLaunchTemplates(ctx, &ec2.DescribeLaunchTemplatesInput{}) + if err != nil { + return count, err } - return count, err + + return len(lts), err } func (lt *LaunchTemplate) Get(ctx context.Context, clusterName string) (names []string, err error) { - var nextToken *string - for { - out, err := lt.ec2Client.DescribeLaunchTemplates(ctx, &ec2.DescribeLaunchTemplatesInput{ - Filters: []ec2types.Filter{ - { - Name: lo.ToPtr("tag:" + karpenterLaunchTemplateTag), - Values: []string{clusterName}, - }, + lts, err := lt.getAllLaunchTemplates(ctx, &ec2.DescribeLaunchTemplatesInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("tag:" + karpenterLaunchTemplateTag), + Values: []string{clusterName}, }, - NextToken: nextToken, - }) - if err != nil { - return names, err - } - - for _, launchTemplate := range out.LaunchTemplates { - names = append(names, lo.FromPtr(launchTemplate.LaunchTemplateName)) - } + }, + }) + if err != nil { + return names, err + } - nextToken = out.NextToken - if nextToken == nil { - break - } + for _, launchtemplate := range lts { + names = append(names, lo.FromPtr(launchtemplate.LaunchTemplateName)) } + return names, err } @@ -142,3 +115,17 @@ func (lt *LaunchTemplate) Cleanup(ctx context.Context, names []string) ([]string } return deleted, errs } + +func (lt *LaunchTemplate) getAllLaunchTemplates(ctx context.Context, params *ec2.DescribeLaunchTemplatesInput) (lts []ec2types.LaunchTemplate, err error) { + paginator := ec2.NewDescribeLaunchTemplatesPaginator(lt.ec2Client, params) + + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return lts, err + } + lts = append(lts, page.LaunchTemplates...) + } + + return lts, nil +} diff --git a/test/hack/resource/pkg/resourcetypes/resourcetypes.go b/test/hack/resource/pkg/resourcetypes/resourcetypes.go index ce8a7bfd9441..9757c14fcd26 100644 --- a/test/hack/resource/pkg/resourcetypes/resourcetypes.go +++ b/test/hack/resource/pkg/resourcetypes/resourcetypes.go @@ -20,7 +20,7 @@ import ( ) const ( - karpenterClusterNameTag = "karpenter.sh/managed-by" + karpenterClusterNameTag = "eks:eks-cluster-name" karpenterNodePoolTag = "karpenter.sh/nodepool" karpenterLaunchTemplateTag = "karpenter.k8s.aws/cluster" karpenterSecurityGroupTag = "karpenter.sh/discovery" diff --git a/test/hack/resource/pkg/resourcetypes/securitygroup.go b/test/hack/resource/pkg/resourcetypes/securitygroup.go index e7224ce694dd..d0a4db53ca4f 100644 --- a/test/hack/resource/pkg/resourcetypes/securitygroup.go +++ b/test/hack/resource/pkg/resourcetypes/securitygroup.go @@ -43,96 +43,69 @@ func (sg *SecurityGroup) Global() bool { } func (sg *SecurityGroup) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) { - var nextToken *string - for { - out, err := sg.ec2Client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ - Filters: []ec2types.Filter{ - { - Name: lo.ToPtr("group-name"), - Values: []string{"security-group-drift"}, - }, + sgs, err := sg.getAllSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("group-name"), + Values: []string{"security-group-drift"}, }, - NextToken: nextToken, + }, + }) + if err != nil { + return ids, err + } + + for _, sgroup := range sgs { + clusterName, found := lo.Find(sgroup.Tags, func(tag ec2types.Tag) bool { + return *tag.Key == karpenterTestingTag }) - if err != nil { - return ids, err + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue } - - for _, sgroup := range out.SecurityGroups { - clusterName, found := lo.Find(sgroup.Tags, func(tag ec2types.Tag) bool { - return *tag.Key == karpenterTestingTag - }) - if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { - continue - } - creationDate, found := lo.Find(sgroup.Tags, func(tag ec2types.Tag) bool { - return *tag.Key == "creation-date" - }) - if !found { - continue - } - time, err := time.Parse(time.RFC3339, *creationDate.Value) - if err != nil { - continue - } - if time.Before(expirationTime) { - ids = append(ids, lo.FromPtr(sgroup.GroupId)) - } + creationDate, found := lo.Find(sgroup.Tags, func(tag ec2types.Tag) bool { + return *tag.Key == "creation-date" + }) + if !found { + continue } - - nextToken = out.NextToken - if nextToken == nil { - break + time, err := time.Parse(time.RFC3339, *creationDate.Value) + if err != nil { + continue + } + if time.Before(expirationTime) { + ids = append(ids, lo.FromPtr(sgroup.GroupId)) } } + return ids, err } func (sg *SecurityGroup) CountAll(ctx context.Context) (count int, err error) { - var nextToken *string - for { - out, err := sg.ec2Client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ - NextToken: nextToken, - }) - if err != nil { - return count, err - } - - count += len(out.SecurityGroups) - - nextToken = out.NextToken - if nextToken == nil { - break - } + sgs, err := sg.getAllSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{}) + if err != nil { + return count, err } - return count, err + + return len(sgs), err } func (sg *SecurityGroup) Get(ctx context.Context, clusterName string) (ids []string, err error) { - var nextToken *string - for { - out, err := sg.ec2Client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ - Filters: []ec2types.Filter{ - { - Name: lo.ToPtr("tag:" + karpenterSecurityGroupTag), - Values: []string{clusterName}, - }, + sgs, err := sg.getAllSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("tag:" + karpenterSecurityGroupTag), + Values: []string{clusterName}, }, - NextToken: nextToken, - }) - if err != nil { - return ids, err - } - - for _, sgroup := range out.SecurityGroups { - ids = append(ids, lo.FromPtr(sgroup.GroupId)) - } + }, + }) + if err != nil { + return ids, err + } - nextToken = out.NextToken - if nextToken == nil { - break - } + for _, sgroup := range sgs { + ids = append(ids, lo.FromPtr(sgroup.GroupId)) } + return ids, err } @@ -154,3 +127,17 @@ func (sg *SecurityGroup) Cleanup(ctx context.Context, ids []string) ([]string, e return deleted, errs } + +func (sg *SecurityGroup) getAllSecurityGroups(ctx context.Context, params *ec2.DescribeSecurityGroupsInput) (sgs []ec2types.SecurityGroup, err error) { + paginator := ec2.NewDescribeSecurityGroupsPaginator(sg.ec2Client, params) + + for paginator.HasMorePages() { + out, err := paginator.NextPage(ctx) + if err != nil { + return nil, err + } + sgs = append(sgs, out.SecurityGroups...) + } + + return sgs, nil +} diff --git a/test/hack/resource/pkg/resourcetypes/stack.go b/test/hack/resource/pkg/resourcetypes/stack.go index fed6ecece85f..ac73618e3436 100644 --- a/test/hack/resource/pkg/resourcetypes/stack.go +++ b/test/hack/resource/pkg/resourcetypes/stack.go @@ -42,80 +42,51 @@ func (s *Stack) Global() bool { } func (s *Stack) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (names []string, err error) { - var nextToken *string - for { - out, err := s.cloudFormationClient.DescribeStacks(ctx, &cloudformation.DescribeStacksInput{ - NextToken: nextToken, - }) - if err != nil { - return names, err - } + stacks, err := s.getAllStacks(ctx) + if err != nil { + return names, err + } - stacks := lo.Reject(out.Stacks, func(s cloudformationtypes.Stack, _ int) bool { - return s.StackStatus == cloudformationtypes.StackStatusDeleteComplete || - s.StackStatus == cloudformationtypes.StackStatusDeleteInProgress + activeStacks := lo.Reject(stacks, func(s cloudformationtypes.Stack, _ int) bool { + return s.StackStatus == cloudformationtypes.StackStatusDeleteComplete || + s.StackStatus == cloudformationtypes.StackStatusDeleteInProgress + }) + for _, stack := range activeStacks { + clusterName, found := lo.Find(stack.Tags, func(tag cloudformationtypes.Tag) bool { + return *tag.Key == karpenterTestingTag }) - for _, stack := range stacks { - clusterName, found := lo.Find(stack.Tags, func(tag cloudformationtypes.Tag) bool { - return *tag.Key == karpenterTestingTag - }) - if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { - continue - } - if _, found := lo.Find(stack.Tags, func(t cloudformationtypes.Tag) bool { - return lo.FromPtr(t.Key) == karpenterTestingTag || lo.FromPtr(t.Key) == githubRunURLTag - }); found && lo.FromPtr(stack.CreationTime).Before(expirationTime) { - names = append(names, lo.FromPtr(stack.StackName)) - } + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue } - - nextToken = out.NextToken - if nextToken == nil { - break + if _, found := lo.Find(stack.Tags, func(t cloudformationtypes.Tag) bool { + return lo.FromPtr(t.Key) == karpenterTestingTag || lo.FromPtr(t.Key) == githubRunURLTag + }); found && lo.FromPtr(stack.CreationTime).Before(expirationTime) { + names = append(names, lo.FromPtr(stack.StackName)) } } return names, err } func (s *Stack) CountAll(ctx context.Context) (count int, err error) { - var nextToken *string - for { - out, err := s.cloudFormationClient.DescribeStacks(ctx, &cloudformation.DescribeStacksInput{ - NextToken: nextToken, - }) - if err != nil { - return count, err - } - - count += len(out.Stacks) - - nextToken = out.NextToken - if nextToken == nil { - break - } + stacks, err := s.getAllStacks(ctx) + if err != nil { + return count, err } - return count, nil + + return len(stacks), nil } func (s *Stack) Get(ctx context.Context, clusterName string) (names []string, err error) { - var nextToken *string - for { - out, err := s.cloudFormationClient.DescribeStacks(ctx, &cloudformation.DescribeStacksInput{ - NextToken: nextToken, - }) - if err != nil { - return names, err - } - for _, stack := range out.Stacks { - if _, found := lo.Find(stack.Tags, func(t cloudformationtypes.Tag) bool { - return lo.FromPtr(t.Key) == karpenterTestingTag && lo.FromPtr(t.Value) == clusterName - }); found { - names = append(names, lo.FromPtr(stack.StackName)) - } - } - nextToken = out.NextToken - if nextToken == nil { - break + stacks, err := s.getAllStacks(ctx) + if err != nil { + return names, err + } + + for _, stack := range stacks { + if _, found := lo.Find(stack.Tags, func(t cloudformationtypes.Tag) bool { + return lo.FromPtr(t.Key) == karpenterTestingTag && lo.FromPtr(t.Value) == clusterName + }); found { + names = append(names, lo.FromPtr(stack.StackName)) } } return names, nil @@ -138,3 +109,17 @@ func (s *Stack) Cleanup(ctx context.Context, names []string) ([]string, error) { } return deleted, errs } + +func (s *Stack) getAllStacks(ctx context.Context) (stacks []cloudformationtypes.Stack, err error) { + paginator := cloudformation.NewDescribeStacksPaginator(s.cloudFormationClient, &cloudformation.DescribeStacksInput{}) + + for paginator.HasMorePages() { + out, err := paginator.NextPage(ctx) + if err != nil { + return stacks, err + } + stacks = append(stacks, out.Stacks...) + } + + return stacks, nil +} diff --git a/test/hack/resource/pkg/resourcetypes/vpc_endpoint.go b/test/hack/resource/pkg/resourcetypes/vpc_endpoint.go index a87d8a3b599c..b12ea02784bc 100644 --- a/test/hack/resource/pkg/resourcetypes/vpc_endpoint.go +++ b/test/hack/resource/pkg/resourcetypes/vpc_endpoint.go @@ -40,83 +40,60 @@ func (v *VPCEndpoint) Global() bool { return false } -func (v *VPCEndpoint) Get(ctx context.Context, clusterName string) (ids []string, err error) { - var nextToken *string - for { - out, err := v.ec2Client.DescribeVpcEndpoints(ctx, &ec2.DescribeVpcEndpointsInput{ - Filters: []ec2types.Filter{ - { - Name: lo.ToPtr("tag:" + karpenterTestingTag), - Values: []string{clusterName}, - }, +func (v *VPCEndpoint) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) { + endpoints, err := v.getAllVpcEndpoints(ctx, &ec2.DescribeVpcEndpointsInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("tag-key"), + Values: []string{karpenterTestingTag}, }, - NextToken: nextToken, + }, + }) + if err != nil { + return ids, err + } + + for _, endpoint := range endpoints { + clusterName, found := lo.Find(endpoint.Tags, func(tag ec2types.Tag) bool { + return *tag.Key == k8sClusterTag }) - if err != nil { - return ids, err + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue } - for _, endpoint := range out.VpcEndpoints { + if endpoint.CreationTimestamp.Before(expirationTime) { ids = append(ids, lo.FromPtr(endpoint.VpcEndpointId)) } - nextToken = out.NextToken - if nextToken == nil { - break - } } + return ids, err } func (v *VPCEndpoint) CountAll(ctx context.Context) (count int, err error) { - var nextToken *string - for { - out, err := v.ec2Client.DescribeVpcEndpoints(ctx, &ec2.DescribeVpcEndpointsInput{ - NextToken: nextToken, - }) - if err != nil { - return count, err - } - - count += len(out.VpcEndpoints) - - nextToken = out.NextToken - if nextToken == nil { - break - } + endpoints, err := v.getAllVpcEndpoints(ctx, &ec2.DescribeVpcEndpointsInput{}) + if err != nil { + return count, err } - return count, err + + return len(endpoints), err } -func (v *VPCEndpoint) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) { - var nextToken *string - for { - out, err := v.ec2Client.DescribeVpcEndpoints(ctx, &ec2.DescribeVpcEndpointsInput{ - Filters: []ec2types.Filter{ - { - Name: lo.ToPtr("tag-key"), - Values: []string{karpenterTestingTag}, - }, +func (v *VPCEndpoint) Get(ctx context.Context, clusterName string) (ids []string, err error) { + endpoints, err := v.getAllVpcEndpoints(ctx, &ec2.DescribeVpcEndpointsInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("tag:" + karpenterTestingTag), + Values: []string{clusterName}, }, - NextToken: nextToken, - }) - if err != nil { - return ids, err - } - for _, endpoint := range out.VpcEndpoints { - clusterName, found := lo.Find(endpoint.Tags, func(tag ec2types.Tag) bool { - return *tag.Key == k8sClusterTag - }) - if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { - continue - } - if endpoint.CreationTimestamp.Before(expirationTime) { - ids = append(ids, lo.FromPtr(endpoint.VpcEndpointId)) - } - } - nextToken = out.NextToken - if nextToken == nil { - break - } + }, + }) + if err != nil { + return ids, err + } + + for _, endpoint := range endpoints { + ids = append(ids, lo.FromPtr(endpoint.VpcEndpointId)) } + return ids, err } @@ -129,3 +106,17 @@ func (v *VPCEndpoint) Cleanup(ctx context.Context, ids []string) ([]string, erro } return ids, nil } + +func (v *VPCEndpoint) getAllVpcEndpoints(ctx context.Context, params *ec2.DescribeVpcEndpointsInput) (endpoints []ec2types.VpcEndpoint, err error) { + paginator := ec2.NewDescribeVpcEndpointsPaginator(v.ec2Client, params) + + for paginator.HasMorePages() { + out, err := paginator.NextPage(ctx) + if err != nil { + return endpoints, err + } + endpoints = append(endpoints, out.VpcEndpoints...) + } + + return endpoints, nil +} diff --git a/test/hack/resource/pkg/resourcetypes/vpc_peering_connection.go b/test/hack/resource/pkg/resourcetypes/vpc_peering_connection.go index b7c9648c609c..85e87a09a024 100644 --- a/test/hack/resource/pkg/resourcetypes/vpc_peering_connection.go +++ b/test/hack/resource/pkg/resourcetypes/vpc_peering_connection.go @@ -40,83 +40,63 @@ func (v *VPCPeeringConnection) Global() bool { return false } -func (v *VPCPeeringConnection) Get(ctx context.Context, clusterName string) (ids []string, err error) { - var nextToken *string - for { - out, err := v.ec2Client.DescribeVpcPeeringConnections(ctx, &ec2.DescribeVpcPeeringConnectionsInput{ - Filters: []ec2types.Filter{ - { - Name: lo.ToPtr("tag:" + karpenterTestingTag), - Values: []string{clusterName}, - }, +func (v *VPCPeeringConnection) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) { + connections, err := v.getAllVpcPeeringConnections(ctx, &ec2.DescribeVpcPeeringConnectionsInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("tag-key"), + Values: []string{karpenterTestingTag}, }, - NextToken: nextToken, + }, + }) + if err != nil { + return ids, err + } + + for _, connection := range connections { + clusterName, found := lo.Find(connection.Tags, func(tag ec2types.Tag) bool { + return *tag.Key == k8sClusterTag }) - if err != nil { - return ids, err + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue } - for _, connection := range out.VpcPeeringConnections { - ids = append(ids, lo.FromPtr(connection.VpcPeeringConnectionId)) + if connection.ExpirationTime == nil { // Connection is non-pending + continue } - nextToken = out.NextToken - if nextToken == nil { - break + if connection.ExpirationTime.Before(expirationTime) { + ids = append(ids, lo.FromPtr(connection.VpcPeeringConnectionId)) } } + return ids, err } func (v *VPCPeeringConnection) CountAll(ctx context.Context) (count int, err error) { - var nextToken *string - for { - out, err := v.ec2Client.DescribeVpcPeeringConnections(ctx, &ec2.DescribeVpcPeeringConnectionsInput{ - NextToken: nextToken, - }) - if err != nil { - return count, err - } - - count += len(out.VpcPeeringConnections) - - nextToken = out.NextToken - if nextToken == nil { - break - } + connections, err := v.getAllVpcPeeringConnections(ctx, &ec2.DescribeVpcPeeringConnectionsInput{}) + if err != nil { + return count, err } - return count, err + + return len(connections), err } -func (v *VPCPeeringConnection) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) { - var nextToken *string - for { - out, err := v.ec2Client.DescribeVpcPeeringConnections(ctx, &ec2.DescribeVpcPeeringConnectionsInput{ - Filters: []ec2types.Filter{ - { - Name: lo.ToPtr("tag-key"), - Values: []string{karpenterTestingTag}, - }, +func (v *VPCPeeringConnection) Get(ctx context.Context, clusterName string) (ids []string, err error) { + connections, err := v.getAllVpcPeeringConnections(ctx, &ec2.DescribeVpcPeeringConnectionsInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("tag:" + karpenterTestingTag), + Values: []string{clusterName}, }, - NextToken: nextToken, - }) - if err != nil { - return ids, err - } - for _, connection := range out.VpcPeeringConnections { - clusterName, found := lo.Find(connection.Tags, func(tag ec2types.Tag) bool { - return *tag.Key == k8sClusterTag - }) - if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { - continue - } - if connection.ExpirationTime.Before(expirationTime) { - ids = append(ids, lo.FromPtr(connection.VpcPeeringConnectionId)) - } - } - nextToken = out.NextToken - if nextToken == nil { - break - } + }, + }) + if err != nil { + return ids, err + } + + for _, connection := range connections { + ids = append(ids, lo.FromPtr(connection.VpcPeeringConnectionId)) } + return ids, err } @@ -131,3 +111,17 @@ func (v *VPCPeeringConnection) Cleanup(ctx context.Context, ids []string) ([]str } return ids, nil } + +func (v *VPCPeeringConnection) getAllVpcPeeringConnections(ctx context.Context, params *ec2.DescribeVpcPeeringConnectionsInput) (connections []ec2types.VpcPeeringConnection, err error) { + paginator := ec2.NewDescribeVpcPeeringConnectionsPaginator(v.ec2Client, params) + + for paginator.HasMorePages() { + out, err := paginator.NextPage(ctx) + if err != nil { + return nil, err + } + connections = append(connections, out.VpcPeeringConnections...) + } + + return connections, nil +} diff --git a/test/hack/soak/get_clusters.go b/test/hack/soak/get_clusters.go index 4cd9e27481cf..0c954986bbca 100644 --- a/test/hack/soak/get_clusters.go +++ b/test/hack/soak/get_clusters.go @@ -36,9 +36,7 @@ type cluster struct { const expirationTTL = time.Hour * 168 // 7 days -var excludedClustersCleanup = []string{ - "soak-periodic-2785632730", -} +var excludedClustersCleanup = []string{} func main() { ctx := context.Background() diff --git a/test/pkg/debug/events.go b/test/pkg/debug/events.go index 0c15d48ee194..79d8648830e5 100644 --- a/test/pkg/debug/events.go +++ b/test/pkg/debug/events.go @@ -22,7 +22,7 @@ import ( "github.com/samber/lo" "go.uber.org/multierr" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "sigs.k8s.io/controller-runtime/pkg/client" @@ -49,13 +49,13 @@ func (c *EventClient) DumpEvents(ctx context.Context) error { } func (c *EventClient) dumpPodEvents(ctx context.Context) error { - el := &v1.EventList{} + el := &corev1.EventList{} if err := c.kubeClient.List(ctx, el, &client.ListOptions{ FieldSelector: fields.SelectorFromSet(map[string]string{"involvedObject.kind": "Pod"}), }); err != nil { return err } - events := lo.Filter(filterTestEvents(el.Items, c.start), func(e v1.Event, _ int) bool { + events := lo.Filter(filterTestEvents(el.Items, c.start), func(e corev1.Event, _ int) bool { return e.InvolvedObject.Namespace != "kube-system" }) for k, v := range coallateEvents(events) { @@ -65,7 +65,7 @@ func (c *EventClient) dumpPodEvents(ctx context.Context) error { } func (c *EventClient) dumpNodeEvents(ctx context.Context) error { - el := &v1.EventList{} + el := &corev1.EventList{} if err := c.kubeClient.List(ctx, el, &client.ListOptions{ FieldSelector: fields.SelectorFromSet(map[string]string{"involvedObject.kind": "Node"}), }); err != nil { @@ -77,8 +77,8 @@ func (c *EventClient) dumpNodeEvents(ctx context.Context) error { return nil } -func filterTestEvents(events []v1.Event, startTime time.Time) []v1.Event { - return lo.Filter(events, func(e v1.Event, _ int) bool { +func filterTestEvents(events []corev1.Event, startTime time.Time) []corev1.Event { + return lo.Filter(events, func(e corev1.Event, _ int) bool { if !e.EventTime.IsZero() { if e.EventTime.BeforeTime(&metav1.Time{Time: startTime}) { return false @@ -90,13 +90,13 @@ func filterTestEvents(events []v1.Event, startTime time.Time) []v1.Event { }) } -func coallateEvents(events []v1.Event) map[v1.ObjectReference]*v1.EventList { - eventMap := map[v1.ObjectReference]*v1.EventList{} +func coallateEvents(events []corev1.Event) map[corev1.ObjectReference]*corev1.EventList { + eventMap := map[corev1.ObjectReference]*corev1.EventList{} for i := range events { elem := events[i] - objectKey := v1.ObjectReference{Kind: elem.InvolvedObject.Kind, Namespace: elem.InvolvedObject.Namespace, Name: elem.InvolvedObject.Name} + objectKey := corev1.ObjectReference{Kind: elem.InvolvedObject.Kind, Namespace: elem.InvolvedObject.Namespace, Name: elem.InvolvedObject.Name} if _, ok := eventMap[objectKey]; !ok { - eventMap[objectKey] = &v1.EventList{} + eventMap[objectKey] = &corev1.EventList{} } eventMap[objectKey].Items = append(eventMap[objectKey].Items, elem) } @@ -105,7 +105,7 @@ func coallateEvents(events []v1.Event) map[v1.ObjectReference]*v1.EventList { // Partially copied from // https://github.com/kubernetes/kubernetes/blob/04ee339c7a4d36b4037ce3635993e2a9e395ebf3/staging/src/k8s.io/kubectl/pkg/describe/describe.go#L4232 -func getEventInformation(o v1.ObjectReference, el *v1.EventList) string { +func getEventInformation(o corev1.ObjectReference, el *corev1.EventList) string { sb := strings.Builder{} sb.WriteString(fmt.Sprintf("------- %s/%s%s EVENTS -------\n", strings.ToLower(o.Kind), lo.Ternary(o.Namespace != "", o.Namespace+"/", ""), o.Name)) diff --git a/test/pkg/debug/monitor.go b/test/pkg/debug/monitor.go index 4d807c0a11d9..cf331fd4bdc9 100644 --- a/test/pkg/debug/monitor.go +++ b/test/pkg/debug/monitor.go @@ -18,18 +18,14 @@ import ( "context" "sync" - "github.com/go-logr/zapr" + "github.com/awslabs/operatorpkg/controller" "github.com/samber/lo" "k8s.io/client-go/rest" - "knative.dev/pkg/logging" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - ctrl "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/server" - - "sigs.k8s.io/karpenter/pkg/operator/controller" - "sigs.k8s.io/karpenter/pkg/operator/scheme" ) type Monitor struct { @@ -40,22 +36,14 @@ type Monitor struct { } func New(ctx context.Context, config *rest.Config, kubeClient client.Client) *Monitor { - logger := logging.FromContext(ctx) - ctrl.SetLogger(zapr.NewLogger(logger.Desugar())) + log.SetLogger(log.FromContext(ctx)) mgr := lo.Must(controllerruntime.NewManager(config, controllerruntime.Options{ - Scheme: scheme.Scheme, - BaseContext: func() context.Context { - ctx := context.Background() - ctx = logging.WithLogger(ctx, logger) - logger.WithOptions() - return ctx - }, Metrics: server.Options{ BindAddress: "0", }, })) for _, c := range newControllers(kubeClient) { - lo.Must0(c.Builder(ctx, mgr).Complete(c), "failed to register controller") + lo.Must0(c.Register(ctx, mgr), "failed to register controller") } ctx, cancel := context.WithCancel(ctx) // this context is only meant for monitor start/stop return &Monitor{ diff --git a/test/pkg/debug/node.go b/test/pkg/debug/node.go index 5fd1d0e3f4ff..3549d743945e 100644 --- a/test/pkg/debug/node.go +++ b/test/pkg/debug/node.go @@ -19,7 +19,7 @@ import ( "fmt" "time" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -29,9 +29,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" nodeutils "sigs.k8s.io/karpenter/pkg/utils/node" ) @@ -45,12 +44,8 @@ func NewNodeController(kubeClient client.Client) *NodeController { } } -func (c *NodeController) Name() string { - return "node" -} - func (c *NodeController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - n := &v1.Node{} + n := &corev1.Node{} if err := c.kubeClient.Get(ctx, req.NamespacedName, n); err != nil { if errors.IsNotFound(err) { fmt.Printf("[DELETED %s] NODE %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.String()) @@ -61,26 +56,27 @@ func (c *NodeController) Reconcile(ctx context.Context, req reconcile.Request) ( return reconcile.Result{}, nil } -func (c *NodeController) GetInfo(ctx context.Context, n *v1.Node) string { +func (c *NodeController) GetInfo(ctx context.Context, n *corev1.Node) string { pods, _ := nodeutils.GetPods(ctx, c.kubeClient, n) - return fmt.Sprintf("ready=%s schedulable=%t initialized=%s pods=%d taints=%v", nodeutils.GetCondition(n, v1.NodeReady).Status, !n.Spec.Unschedulable, n.Labels[v1beta1.NodeInitializedLabelKey], len(pods), n.Spec.Taints) + return fmt.Sprintf("ready=%s schedulable=%t initialized=%s pods=%d taints=%v", nodeutils.GetCondition(n, corev1.NodeReady).Status, !n.Spec.Unschedulable, n.Labels[karpv1.NodeInitializedLabelKey], len(pods), n.Spec.Taints) } -func (c *NodeController) Builder(ctx context.Context, m manager.Manager) corecontroller.Builder { - return corecontroller.Adapt(controllerruntime. - NewControllerManagedBy(m). - For(&v1.Node{}). +func (c *NodeController) Register(ctx context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("node"). + For(&corev1.Node{}). WithEventFilter(predicate.And( predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { - oldNode := e.ObjectOld.(*v1.Node) - newNode := e.ObjectNew.(*v1.Node) + oldNode := e.ObjectOld.(*corev1.Node) + newNode := e.ObjectNew.(*corev1.Node) return c.GetInfo(ctx, oldNode) != c.GetInfo(ctx, newNode) }, }, predicate.NewPredicateFuncs(func(o client.Object) bool { - return o.GetLabels()[v1beta1.NodePoolLabelKey] != "" + return o.GetLabels()[karpv1.NodePoolLabelKey] != "" }), )). - WithOptions(controller.Options{MaxConcurrentReconciles: 10})) + WithOptions(controller.Options{MaxConcurrentReconciles: 10}). + Complete(c) } diff --git a/test/pkg/debug/nodeclaim.go b/test/pkg/debug/nodeclaim.go index 926f52eed3c2..ccbf67db933e 100644 --- a/test/pkg/debug/nodeclaim.go +++ b/test/pkg/debug/nodeclaim.go @@ -28,8 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" ) type NodeClaimController struct { @@ -42,12 +41,8 @@ func NewNodeClaimController(kubeClient client.Client) *NodeClaimController { } } -func (c *NodeClaimController) Name() string { - return "nodeclaim" -} - func (c *NodeClaimController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - nc := &corev1beta1.NodeClaim{} + nc := &karpv1.NodeClaim{} if err := c.kubeClient.Get(ctx, req.NamespacedName, nc); err != nil { if errors.IsNotFound(err) { fmt.Printf("[DELETED %s] NODECLAIM %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.String()) @@ -58,25 +53,26 @@ func (c *NodeClaimController) Reconcile(ctx context.Context, req reconcile.Reque return reconcile.Result{}, nil } -func (c *NodeClaimController) GetInfo(nc *corev1beta1.NodeClaim) string { +func (c *NodeClaimController) GetInfo(nc *karpv1.NodeClaim) string { return fmt.Sprintf("ready=%t launched=%t registered=%t initialized=%t", - nc.StatusConditions().IsHappy(), - nc.StatusConditions().GetCondition(corev1beta1.Launched).IsTrue(), - nc.StatusConditions().GetCondition(corev1beta1.Registered).IsTrue(), - nc.StatusConditions().GetCondition(corev1beta1.Initialized).IsTrue(), + nc.StatusConditions().Root().IsTrue(), + nc.StatusConditions().Get(karpv1.ConditionTypeLaunched).IsTrue(), + nc.StatusConditions().Get(karpv1.ConditionTypeRegistered).IsTrue(), + nc.StatusConditions().Get(karpv1.ConditionTypeInitialized).IsTrue(), ) } -func (c *NodeClaimController) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { - return corecontroller.Adapt(controllerruntime. - NewControllerManagedBy(m). - For(&corev1beta1.NodeClaim{}). +func (c *NodeClaimController) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("nodeclaim"). + For(&karpv1.NodeClaim{}). WithEventFilter(predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { - oldNodeClaim := e.ObjectOld.(*corev1beta1.NodeClaim) - newNodeClaim := e.ObjectNew.(*corev1beta1.NodeClaim) + oldNodeClaim := e.ObjectOld.(*karpv1.NodeClaim) + newNodeClaim := e.ObjectNew.(*karpv1.NodeClaim) return c.GetInfo(oldNodeClaim) != c.GetInfo(newNodeClaim) }, }). - WithOptions(controller.Options{MaxConcurrentReconciles: 10})) + WithOptions(controller.Options{MaxConcurrentReconciles: 10}). + Complete(c) } diff --git a/test/pkg/debug/pod.go b/test/pkg/debug/pod.go index a2bea9f21bae..8545b6908f2f 100644 --- a/test/pkg/debug/pod.go +++ b/test/pkg/debug/pod.go @@ -21,7 +21,7 @@ import ( "time" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,7 +31,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" "sigs.k8s.io/karpenter/pkg/utils/pod" ) @@ -45,12 +44,8 @@ func NewPodController(kubeClient client.Client) *PodController { } } -func (c *PodController) Name() string { - return "pod" -} - func (c *PodController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - p := &v1.Pod{} + p := &corev1.Pod{} if err := c.kubeClient.Get(ctx, req.NamespacedName, p); err != nil { if errors.IsNotFound(err) { fmt.Printf("[DELETED %s] POD %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.String()) @@ -61,7 +56,7 @@ func (c *PodController) Reconcile(ctx context.Context, req reconcile.Request) (r return reconcile.Result{}, nil } -func (c *PodController) GetInfo(p *v1.Pod) string { +func (c *PodController) GetInfo(p *corev1.Pod) string { var containerInfo strings.Builder for _, c := range p.Status.ContainerStatuses { if containerInfo.Len() > 0 { @@ -73,15 +68,15 @@ func (c *PodController) GetInfo(p *v1.Pod) string { pod.IsProvisionable(p), p.Status.Phase, p.Spec.NodeName, p.OwnerReferences, containerInfo.String()) } -func (c *PodController) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { - return corecontroller.Adapt(controllerruntime. - NewControllerManagedBy(m). - For(&v1.Pod{}). +func (c *PodController) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("pod"). + For(&corev1.Pod{}). WithEventFilter(predicate.And( predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { - oldPod := e.ObjectOld.(*v1.Pod) - newPod := e.ObjectNew.(*v1.Pod) + oldPod := e.ObjectOld.(*corev1.Pod) + newPod := e.ObjectNew.(*corev1.Pod) return c.GetInfo(oldPod) != c.GetInfo(newPod) }, }, @@ -89,5 +84,6 @@ func (c *PodController) Builder(_ context.Context, m manager.Manager) corecontro return o.GetNamespace() != "kube-system" }), )). - WithOptions(controller.Options{MaxConcurrentReconciles: 10})) + WithOptions(controller.Options{MaxConcurrentReconciles: 10}). + Complete(c) } diff --git a/test/pkg/environment/aws/environment.go b/test/pkg/environment/aws/environment.go index e874cf325ce1..8df6765a9915 100644 --- a/test/pkg/environment/aws/environment.go +++ b/test/pkg/environment/aws/environment.go @@ -40,30 +40,22 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/utils/env" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "sigs.k8s.io/karpenter/pkg/operator/scheme" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/sqs" "github.com/aws/karpenter-provider-aws/pkg/test" "github.com/aws/karpenter-provider-aws/test/pkg/environment/common" ) func init() { - lo.Must0(apis.AddToScheme(scheme.Scheme)) - corev1beta1.NormalizedLabels = lo.Assign(corev1beta1.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": corev1.LabelTopologyZone}) + karpv1.NormalizedLabels = lo.Assign(karpv1.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": corev1.LabelTopologyZone}) } var WindowsDefaultImage = "mcr.microsoft.com/oss/kubernetes/pause:3.9" var EphemeralInitContainerImage = "alpine" -// ExcludedInstanceFamilies denotes instance families that have issues during resource registration due to compatibility -// issues with versions of the VPR Resource Controller. -// TODO: jmdeal@ remove a1 from exclusion list once Karpenter implicitly filters a1 instances for AL2023 AMI family (incompatible) -var ExcludedInstanceFamilies = []string{"m7a", "r7a", "c7a", "r7i", "a1"} - type Environment struct { *common.Environment Region string @@ -82,6 +74,13 @@ type Environment struct { ClusterEndpoint string InterruptionQueue string PrivateCluster bool + ZoneInfo []ZoneInfo +} + +type ZoneInfo struct { + Zone string + ZoneID string + ZoneType string } func NewEnvironment(t *testing.T) *Environment { @@ -123,6 +122,14 @@ func NewEnvironment(t *testing.T) *Environment { out := lo.Must(sqsapi.GetQueueUrlWithContext(env.Context, &servicesqs.GetQueueUrlInput{QueueName: aws.String(v)})) awsEnv.SQSProvider = lo.Must(sqs.NewDefaultProvider(sqsapi, lo.FromPtr(out.QueueUrl))) } + // Populate ZoneInfo for all AZs in the region + awsEnv.ZoneInfo = lo.Map(lo.Must(awsEnv.EC2API.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{})).AvailabilityZones, func(zone *ec2.AvailabilityZone, _ int) ZoneInfo { + return ZoneInfo{ + Zone: lo.FromPtr(zone.ZoneName), + ZoneID: lo.FromPtr(zone.ZoneId), + ZoneType: lo.FromPtr(zone.ZoneType), + } + }) return awsEnv } @@ -134,18 +141,18 @@ func GetTimeStreamAPI(session *session.Session) timestreamwriteiface.TimestreamW return &NoOpTimeStreamAPI{} } -func (env *Environment) DefaultEC2NodeClass() *v1beta1.EC2NodeClass { +func (env *Environment) DefaultEC2NodeClass() *v1.EC2NodeClass { nodeClass := test.EC2NodeClass() - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2023@latest"}} nodeClass.Spec.Tags = map[string]string{ "testing/cluster": env.ClusterName, } - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, } - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { Tags: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, diff --git a/test/pkg/environment/aws/expectations.go b/test/pkg/environment/aws/expectations.go index 2b69ccd47d64..b40a4c4ae545 100644 --- a/test/pkg/environment/aws/expectations.go +++ b/test/pkg/environment/aws/expectations.go @@ -36,7 +36,7 @@ import ( coretest "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" awserrors "github.com/aws/karpenter-provider-aws/pkg/errors" . "github.com/onsi/ginkgo/v2" @@ -137,7 +137,7 @@ func (env *Environment) EventuallyExpectInstanceProfileExists(profileName string // GetInstanceProfileName gets the string for the profile name based on the cluster name, region and the NodeClass name. // The length of this string can never exceed the maximum instance profile name limit of 128 characters. -func (env *Environment) GetInstanceProfileName(nodeClass *v1beta1.EC2NodeClass) string { +func (env *Environment) GetInstanceProfileName(nodeClass *v1.EC2NodeClass) string { return fmt.Sprintf("%s_%d", env.ClusterName, lo.Must(hashstructure.Hash(fmt.Sprintf("%s%s", env.Region, nodeClass.Name), hashstructure.FormatV2, nil))) } @@ -210,14 +210,6 @@ func (env *Environment) GetSpotInstanceRequest(id *string) *ec2.SpotInstanceRequ return siro.SpotInstanceRequests[0] } -// GetZones returns all available zones mapped from zone -> zone type -func (env *Environment) GetZones() map[string]string { - output := lo.Must(env.EC2API.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{})) - return lo.Associate(output.AvailabilityZones, func(zone *ec2.AvailabilityZone) (string, string) { - return lo.FromPtr(zone.ZoneName), lo.FromPtr(zone.ZoneType) - }) -} - // GetSubnets returns all subnets matching the label selector // mapped from AZ -> {subnet-ids...} func (env *Environment) GetSubnets(tags map[string]string) map[string][]string { @@ -243,10 +235,11 @@ func (env *Environment) GetSubnets(tags map[string]string) map[string][]string { type SubnetInfo struct { Name string ID string + ZoneInfo } -// GetSubnetNameAndIds returns all subnets matching the label selector -func (env *Environment) GetSubnetNameAndIds(tags map[string]string) []SubnetInfo { +// GetSubnetInfo returns all subnets matching the label selector +func (env *Environment) GetSubnetInfo(tags map[string]string) []SubnetInfo { var filters []*ec2.Filter for key, val := range tags { filters = append(filters, &ec2.Filter{ @@ -261,6 +254,11 @@ func (env *Environment) GetSubnetNameAndIds(tags map[string]string) []SubnetInfo if tag, ok := lo.Find(s.Tags, func(t *ec2.Tag) bool { return aws.StringValue(t.Key) == "Name" }); ok { elem.Name = aws.StringValue(tag.Value) } + if info, ok := lo.Find(env.ZoneInfo, func(info ZoneInfo) bool { + return aws.StringValue(s.AvailabilityZone) == info.Zone + }); ok { + elem.ZoneInfo = info + } return elem }) return true @@ -327,7 +325,15 @@ func (env *Environment) ExpectParsedProviderID(providerID string) string { return providerIDSplit[len(providerIDSplit)-1] } -func (env *Environment) GetK8sVersion(offset int) string { +func (env *Environment) K8sVersion() string { + GinkgoHelper() + + return env.K8sVersionWithOffset(0) +} + +func (env *Environment) K8sVersionWithOffset(offset int) string { + GinkgoHelper() + serverVersion, err := env.KubeClient.Discovery().ServerVersion() Expect(err).To(BeNil()) minorVersion, err := strconv.Atoi(strings.TrimSuffix(serverVersion.Minor, "+")) @@ -338,18 +344,19 @@ func (env *Environment) GetK8sVersion(offset int) string { return fmt.Sprintf("%s.%d", serverVersion.Major, minorVersion-offset) } -func (env *Environment) GetK8sMinorVersion(offset int) (int, error) { - version, err := strconv.Atoi(strings.Split(env.GetK8sVersion(offset), ".")[1]) - if err != nil { - return 0, err - } - return version, nil +func (env *Environment) K8sMinorVersion() int { + GinkgoHelper() + + version, err := strconv.Atoi(strings.Split(env.K8sVersion(), ".")[1]) + Expect(err).ToNot(HaveOccurred()) + return version } -func (env *Environment) GetCustomAMI(amiPath string, versionOffset int) string { - version := env.GetK8sVersion(versionOffset) +func (env *Environment) GetAMIBySSMPath(ssmPath string) string { + GinkgoHelper() + parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ - Name: aws.String(fmt.Sprintf(amiPath, version)), + Name: aws.String(ssmPath), }) Expect(err).To(BeNil()) return *parameter.Parameter.Value diff --git a/test/pkg/environment/aws/setup.go b/test/pkg/environment/aws/setup.go index 99743b6cb7eb..92deae58ad25 100644 --- a/test/pkg/environment/aws/setup.go +++ b/test/pkg/environment/aws/setup.go @@ -15,17 +15,17 @@ limitations under the License. package aws import ( - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" ) -var persistedSettings []v1.EnvVar +var persistedSettings []corev1.EnvVar var ( CleanableObjects = []client.Object{ - &v1beta1.EC2NodeClass{}, + &v1.EC2NodeClass{}, } ) diff --git a/test/pkg/environment/common/environment.go b/test/pkg/environment/common/environment.go index 908a734904be..8f60ecd08f0d 100644 --- a/test/pkg/environment/common/environment.go +++ b/test/pkg/environment/common/environment.go @@ -23,27 +23,27 @@ import ( "testing" "time" + "github.com/awslabs/operatorpkg/object" "github.com/onsi/gomega" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" - loggingtesting "knative.dev/pkg/logging/testing" + + . "sigs.k8s.io/karpenter/pkg/utils/testing" //nolint:stylecheck + "knative.dev/pkg/system" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - coreapis "sigs.k8s.io/karpenter/pkg/apis" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/operator" coretest "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" ) type ContextKey string @@ -65,7 +65,7 @@ type Environment struct { } func NewEnvironment(t *testing.T) *Environment { - ctx := loggingtesting.TestContextWithLogger(t) + ctx := TestContextWithLogger(t) ctx, cancel := context.WithCancel(ctx) config := NewConfig() client := NewClient(ctx, config) @@ -75,7 +75,7 @@ func NewEnvironment(t *testing.T) *Environment { ctx = context.WithValue(ctx, GitRefContextKey, val) } - gomega.SetDefaultEventuallyTimeout(5 * time.Minute) + gomega.SetDefaultEventuallyTimeout(16 * time.Minute) gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) return &Environment{ Context: ctx, @@ -100,33 +100,28 @@ func NewConfig() *rest.Config { } func NewClient(ctx context.Context, config *rest.Config) client.Client { - scheme := runtime.NewScheme() - lo.Must0(clientgoscheme.AddToScheme(scheme)) - lo.Must0(apis.AddToScheme(scheme)) - lo.Must0(coreapis.AddToScheme(scheme)) - - cache := lo.Must(cache.New(config, cache.Options{Scheme: scheme})) - lo.Must0(cache.IndexField(ctx, &v1.Pod{}, "spec.nodeName", func(o client.Object) []string { - pod := o.(*v1.Pod) + cache := lo.Must(cache.New(config, cache.Options{Scheme: scheme.Scheme})) + lo.Must0(cache.IndexField(ctx, &corev1.Pod{}, "spec.nodeName", func(o client.Object) []string { + pod := o.(*corev1.Pod) return []string{pod.Spec.NodeName} })) - lo.Must0(cache.IndexField(ctx, &v1.Event{}, "involvedObject.kind", func(o client.Object) []string { - evt := o.(*v1.Event) + lo.Must0(cache.IndexField(ctx, &corev1.Event{}, "involvedObject.kind", func(o client.Object) []string { + evt := o.(*corev1.Event) return []string{evt.InvolvedObject.Kind} })) - lo.Must0(cache.IndexField(ctx, &v1.Node{}, "spec.unschedulable", func(o client.Object) []string { - node := o.(*v1.Node) + lo.Must0(cache.IndexField(ctx, &corev1.Node{}, "spec.unschedulable", func(o client.Object) []string { + node := o.(*corev1.Node) return []string{strconv.FormatBool(node.Spec.Unschedulable)} })) - lo.Must0(cache.IndexField(ctx, &v1.Node{}, "spec.taints[*].karpenter.sh/disruption", func(o client.Object) []string { - node := o.(*v1.Node) - t, _ := lo.Find(node.Spec.Taints, func(t v1.Taint) bool { - return t.Key == corev1beta1.DisruptionTaintKey + lo.Must0(cache.IndexField(ctx, &corev1.Node{}, "spec.taints[*].karpenter.sh/disrupted", func(o client.Object) []string { + node := o.(*corev1.Node) + _, found := lo.Find(node.Spec.Taints, func(t corev1.Taint) bool { + return t.Key == karpv1.DisruptedTaintKey }) - return []string{t.Value} + return []string{lo.Ternary(found, "true", "false")} })) - c := lo.Must(client.New(config, client.Options{Scheme: scheme, Cache: &client.CacheOptions{Reader: cache}})) + c := lo.Must(client.New(config, client.Options{Scheme: scheme.Scheme, Cache: &client.CacheOptions{Reader: cache}})) go func() { lo.Must0(cache.Start(ctx)) @@ -137,54 +132,57 @@ func NewClient(ctx context.Context, config *rest.Config) client.Client { return c } -func (env *Environment) DefaultNodePool(nodeClass *v1beta1.EC2NodeClass) *corev1beta1.NodePool { +func (env *Environment) DefaultNodePool(nodeClass *v1.EC2NodeClass) *karpv1.NodePool { nodePool := coretest.NodePool() - nodePool.Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nodePool.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, } - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Linux)}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Linux)}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpIn, Values: []string{"c", "m", "r"}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceGeneration, - Operator: v1.NodeSelectorOpGt, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceGeneration, + Operator: corev1.NodeSelectorOpGt, Values: []string{"2"}, }, }, // Filter out a1 instance types, which are incompatible with AL2023 AMIs { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceFamily, + Operator: corev1.NodeSelectorOpNotIn, Values: []string{"a1"}, }, }, } - nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{} - nodePool.Spec.Disruption.ExpireAfter.Duration = nil - nodePool.Spec.Limits = corev1beta1.Limits(v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1000"), - v1.ResourceMemory: resource.MustParse("1000Gi"), + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmptyOrUnderutilized + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("Never") + nodePool.Spec.Template.Spec.ExpireAfter.Duration = nil + nodePool.Spec.Limits = karpv1.Limits(corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1000"), + corev1.ResourceMemory: resource.MustParse("1000Gi"), }) return nodePool } diff --git a/test/pkg/environment/common/expectations.go b/test/pkg/environment/common/expectations.go index adbc5cdda0ee..6069b0aee838 100644 --- a/test/pkg/environment/common/expectations.go +++ b/test/pkg/environment/common/expectations.go @@ -28,7 +28,7 @@ import ( "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" coordinationv1 "k8s.io/api/coordination/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,15 +36,15 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/transport" - "knative.dev/pkg/logging" - "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/log" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" pscheduling "sigs.k8s.io/karpenter/pkg/controllers/provisioning/scheduling" "sigs.k8s.io/karpenter/pkg/scheduling" "sigs.k8s.io/karpenter/pkg/test" + coreresources "sigs.k8s.io/karpenter/pkg/utils/resources" ) func (env *Environment) ExpectCreated(objects ...client.Object) { @@ -63,7 +63,7 @@ func (env *Environment) ExpectDeleted(objects ...client.Object) { GinkgoHelper() for _, object := range objects { Eventually(func(g Gomega) { - g.Expect(client.IgnoreNotFound(env.Client.Delete(env, object, client.PropagationPolicy(metav1.DeletePropagationForeground), &client.DeleteOptions{GracePeriodSeconds: ptr.Int64(0)}))).To(Succeed()) + g.Expect(client.IgnoreNotFound(env.Client.Delete(env, object, client.PropagationPolicy(metav1.DeletePropagationForeground), &client.DeleteOptions{GracePeriodSeconds: lo.ToPtr(int64(0))}))).To(Succeed()) }).WithTimeout(time.Second * 10).Should(Succeed()) } } @@ -81,7 +81,7 @@ func (env *Environment) ExpectUpdated(objects ...client.Object) { current := o.DeepCopyObject().(client.Object) g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(current), current)).To(Succeed()) if current.GetResourceVersion() != o.GetResourceVersion() { - logging.FromContext(env).Infof("detected an update to an object (%s) with an outdated resource version, did you get the latest version of the object before patching?", lo.Must(apiutil.GVKForObject(o, env.Client.Scheme()))) + log.FromContext(env).Info(fmt.Sprintf("detected an update to an object (%s) with an outdated resource version, did you get the latest version of the object before patching?", lo.Must(apiutil.GVKForObject(o, env.Client.Scheme())))) } o.SetResourceVersion(current.GetResourceVersion()) g.Expect(env.Client.Update(env.Context, o)).To(Succeed()) @@ -112,18 +112,18 @@ func (env *Environment) ExpectCreatedOrUpdated(objects ...client.Object) { } } -func (env *Environment) ExpectSettings() (res []v1.EnvVar) { +func (env *Environment) ExpectSettings() (res []corev1.EnvVar) { GinkgoHelper() d := &appsv1.Deployment{} Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "kube-system", Name: "karpenter"}, d)).To(Succeed()) Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) - return lo.Map(d.Spec.Template.Spec.Containers[0].Env, func(v v1.EnvVar, _ int) v1.EnvVar { + return lo.Map(d.Spec.Template.Spec.Containers[0].Env, func(v corev1.EnvVar, _ int) corev1.EnvVar { return *v.DeepCopy() }) } -func (env *Environment) ExpectSettingsReplaced(vars ...v1.EnvVar) { +func (env *Environment) ExpectSettingsReplaced(vars ...corev1.EnvVar) { GinkgoHelper() d := &appsv1.Deployment{} @@ -140,7 +140,7 @@ func (env *Environment) ExpectSettingsReplaced(vars ...v1.EnvVar) { } } -func (env *Environment) ExpectSettingsOverridden(vars ...v1.EnvVar) { +func (env *Environment) ExpectSettingsOverridden(vars ...corev1.EnvVar) { GinkgoHelper() d := &appsv1.Deployment{} @@ -149,7 +149,7 @@ func (env *Environment) ExpectSettingsOverridden(vars ...v1.EnvVar) { stored := d.DeepCopy() for _, v := range vars { - if _, i, ok := lo.FindIndexOf(d.Spec.Template.Spec.Containers[0].Env, func(e v1.EnvVar) bool { + if _, i, ok := lo.FindIndexOf(d.Spec.Template.Spec.Containers[0].Env, func(e corev1.EnvVar) bool { return e.Name == v.Name }); ok { d.Spec.Template.Spec.Containers[0].Env[i] = v @@ -164,17 +164,17 @@ func (env *Environment) ExpectSettingsOverridden(vars ...v1.EnvVar) { } } -func (env *Environment) ExpectSettingsRemoved(vars ...v1.EnvVar) { +func (env *Environment) ExpectSettingsRemoved(vars ...corev1.EnvVar) { GinkgoHelper() - varNames := sets.New[string](lo.Map(vars, func(v v1.EnvVar, _ int) string { return v.Name })...) + varNames := sets.New(lo.Map(vars, func(v corev1.EnvVar, _ int) string { return v.Name })...) d := &appsv1.Deployment{} Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "kube-system", Name: "karpenter"}, d)).To(Succeed()) Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) stored := d.DeepCopy() - d.Spec.Template.Spec.Containers[0].Env = lo.Reject(d.Spec.Template.Spec.Containers[0].Env, func(v v1.EnvVar, _ int) bool { + d.Spec.Template.Spec.Containers[0].Env = lo.Reject(d.Spec.Template.Spec.Containers[0].Env, func(v corev1.EnvVar, _ int) bool { return varNames.Has(v.Name) }) if !equality.Semantic.DeepEqual(d, stored) { @@ -184,16 +184,16 @@ func (env *Environment) ExpectSettingsRemoved(vars ...v1.EnvVar) { } } -func (env *Environment) ExpectConfigMapExists(key types.NamespacedName) *v1.ConfigMap { +func (env *Environment) ExpectConfigMapExists(key types.NamespacedName) *corev1.ConfigMap { GinkgoHelper() - cm := &v1.ConfigMap{} + cm := &corev1.ConfigMap{} Expect(env.Client.Get(env, key, cm)).To(Succeed()) return cm } func (env *Environment) ExpectConfigMapDataReplaced(key types.NamespacedName, data ...map[string]string) (changed bool) { GinkgoHelper() - cm := &v1.ConfigMap{ + cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: key.Name, Namespace: key.Namespace, @@ -216,7 +216,7 @@ func (env *Environment) ExpectConfigMapDataReplaced(key types.NamespacedName, da func (env *Environment) ExpectConfigMapDataOverridden(key types.NamespacedName, data ...map[string]string) (changed bool) { GinkgoHelper() - cm := &v1.ConfigMap{ + cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: key.Name, Namespace: key.Namespace, @@ -225,13 +225,8 @@ func (env *Environment) ExpectConfigMapDataOverridden(key types.NamespacedName, err := env.Client.Get(env, key, cm) Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) - stored := cm.DeepCopy() cm.Data = lo.Assign(append([]map[string]string{cm.Data}, data...)...) - // If the data hasn't changed, we can just return and not update anything - if equality.Semantic.DeepEqual(stored, cm) { - return false - } // Update the configMap to update the settings env.ExpectCreatedOrUpdated(cm) return true @@ -269,7 +264,7 @@ func (env *Environment) ExpectExists(obj client.Object) client.Object { return obj } -func (env *Environment) EventuallyExpectBound(pods ...*v1.Pod) { +func (env *Environment) EventuallyExpectBound(pods ...*corev1.Pod) { GinkgoHelper() Eventually(func(g Gomega) { for _, pod := range pods { @@ -279,23 +274,62 @@ func (env *Environment) EventuallyExpectBound(pods ...*v1.Pod) { }).Should(Succeed()) } -func (env *Environment) EventuallyExpectHealthy(pods ...*v1.Pod) { +func (env *Environment) EventuallyExpectHealthy(pods ...*corev1.Pod) { GinkgoHelper() env.EventuallyExpectHealthyWithTimeout(-1, pods...) } -func (env *Environment) EventuallyExpectHealthyWithTimeout(timeout time.Duration, pods ...*v1.Pod) { +func (env *Environment) EventuallyExpectTerminating(pods ...*corev1.Pod) { + GinkgoHelper() + env.EventuallyExpectTerminatingWithTimeout(-1, pods...) +} + +func (env *Environment) EventuallyExpectTerminatingWithTimeout(timeout time.Duration, pods ...*corev1.Pod) { + GinkgoHelper() + Eventually(func(g Gomega) { + for _, pod := range pods { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + g.Expect(pod.DeletionTimestamp.IsZero()).To(BeFalse()) + } + }).WithTimeout(timeout).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectHealthyWithTimeout(timeout time.Duration, pods ...*corev1.Pod) { GinkgoHelper() Eventually(func(g Gomega) { for _, pod := range pods { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) g.Expect(pod.Status.Conditions).To(ContainElement(And( - HaveField("Type", Equal(v1.PodReady)), - HaveField("Status", Equal(v1.ConditionTrue)), + HaveField("Type", Equal(corev1.PodReady)), + HaveField("Status", Equal(corev1.ConditionTrue)), ))) } }).WithTimeout(timeout).Should(Succeed()) +} +func (env *Environment) ConsistentlyExpectTerminatingPods(duration time.Duration, pods ...*corev1.Pod) { + GinkgoHelper() + By(fmt.Sprintf("expecting %d pods to be terminating for %s", len(pods), duration)) + Consistently(func(g Gomega) { + for _, pod := range pods { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + g.Expect(pod.DeletionTimestamp.IsZero()).To(BeFalse()) + } + }, duration.String()).Should(Succeed()) +} + +func (env *Environment) ConsistentlyExpectHealthyPods(duration time.Duration, pods ...*corev1.Pod) { + GinkgoHelper() + By(fmt.Sprintf("expecting %d pods to be ready for %s", len(pods), duration)) + Consistently(func(g Gomega) { + for _, pod := range pods { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + g.Expect(pod.Status.Conditions).To(ContainElement(And( + HaveField("Type", Equal(corev1.PodReady)), + HaveField("Status", Equal(corev1.ConditionTrue)), + ))) + } + }, duration.String()).Should(Succeed()) } func (env *Environment) EventuallyExpectKarpenterRestarted() { @@ -312,7 +346,7 @@ func (env *Environment) ExpectKarpenterLeaseOwnerChanged() { pods := env.ExpectKarpenterPods() Eventually(func(g Gomega) { name := env.ExpectActiveKarpenterPodName() - g.Expect(lo.ContainsBy(pods, func(p *v1.Pod) bool { + g.Expect(lo.ContainsBy(pods, func(p *corev1.Pod) bool { return p.Name == name })).To(BeTrue()) }).Should(Succeed()) @@ -333,29 +367,29 @@ func (env *Environment) EventuallyExpectRollout(name, namespace string) { By("waiting for the newly generated deployment to rollout") Eventually(func(g Gomega) { - podList := &v1.PodList{} + podList := &corev1.PodList{} g.Expect(env.Client.List(env.Context, podList, client.InNamespace(namespace))).To(Succeed()) - pods := lo.Filter(podList.Items, func(p v1.Pod, _ int) bool { + pods := lo.Filter(podList.Items, func(p corev1.Pod, _ int) bool { return p.Annotations["kubectl.kubernetes.io/restartedAt"] == restartedAtAnnotation["kubectl.kubernetes.io/restartedAt"] }) g.Expect(len(pods)).To(BeNumerically("==", lo.FromPtr(deploy.Spec.Replicas))) for _, pod := range pods { g.Expect(pod.Status.Conditions).To(ContainElement(And( - HaveField("Type", Equal(v1.PodReady)), - HaveField("Status", Equal(v1.ConditionTrue)), + HaveField("Type", Equal(corev1.PodReady)), + HaveField("Status", Equal(corev1.ConditionTrue)), ))) - g.Expect(pod.Status.Phase).To(Equal(v1.PodRunning)) + g.Expect(pod.Status.Phase).To(Equal(corev1.PodRunning)) } }).Should(Succeed()) } -func (env *Environment) ExpectKarpenterPods() []*v1.Pod { +func (env *Environment) ExpectKarpenterPods() []*corev1.Pod { GinkgoHelper() - podList := &v1.PodList{} + podList := &corev1.PodList{} Expect(env.Client.List(env.Context, podList, client.MatchingLabels{ "app.kubernetes.io/instance": "karpenter", })).To(Succeed()) - return lo.Map(podList.Items, func(p v1.Pod, _ int) *v1.Pod { return &p }) + return lo.Map(podList.Items, func(p corev1.Pod, _ int) *corev1.Pod { return &p }) } func (env *Environment) ExpectActiveKarpenterPodName() string { @@ -370,11 +404,11 @@ func (env *Environment) ExpectActiveKarpenterPodName() string { return holderArr[0] } -func (env *Environment) ExpectActiveKarpenterPod() *v1.Pod { +func (env *Environment) ExpectActiveKarpenterPod() *corev1.Pod { GinkgoHelper() podName := env.ExpectActiveKarpenterPodName() - pod := &v1.Pod{} + pod := &corev1.Pod{} Expect(env.Client.Get(env.Context, types.NamespacedName{Name: podName, Namespace: "kube-system"}, pod)).To(Succeed()) return pod } @@ -386,12 +420,12 @@ func (env *Environment) EventuallyExpectPendingPodCount(selector labels.Selector }).Should(Succeed()) } -func (env *Environment) EventuallyExpectBoundPodCount(selector labels.Selector, numPods int) []*v1.Pod { +func (env *Environment) EventuallyExpectBoundPodCount(selector labels.Selector, numPods int) []*corev1.Pod { GinkgoHelper() - var res []*v1.Pod + var res []*corev1.Pod Eventually(func(g Gomega) { - res = []*v1.Pod{} - podList := &v1.PodList{} + res = []*corev1.Pod{} + podList := &corev1.PodList{} g.Expect(env.Client.List(env.Context, podList, client.MatchingLabelsSelector{Selector: selector})).To(Succeed()) for i := range podList.Items { if podList.Items[i].Spec.NodeName != "" { @@ -403,15 +437,15 @@ func (env *Environment) EventuallyExpectBoundPodCount(selector labels.Selector, return res } -func (env *Environment) EventuallyExpectHealthyPodCount(selector labels.Selector, numPods int) []*v1.Pod { +func (env *Environment) EventuallyExpectHealthyPodCount(selector labels.Selector, numPods int) []*corev1.Pod { By(fmt.Sprintf("waiting for %d pods matching selector %s to be ready", numPods, selector.String())) GinkgoHelper() return env.EventuallyExpectHealthyPodCountWithTimeout(-1, selector, numPods) } -func (env *Environment) EventuallyExpectHealthyPodCountWithTimeout(timeout time.Duration, selector labels.Selector, numPods int) []*v1.Pod { +func (env *Environment) EventuallyExpectHealthyPodCountWithTimeout(timeout time.Duration, selector labels.Selector, numPods int) []*corev1.Pod { GinkgoHelper() - var pods []*v1.Pod + var pods []*corev1.Pod Eventually(func(g Gomega) { pods = env.Monitor.RunningPods(selector) g.Expect(pods).To(HaveLen(numPods)) @@ -419,10 +453,10 @@ func (env *Environment) EventuallyExpectHealthyPodCountWithTimeout(timeout time. return pods } -func (env *Environment) ExpectPodsMatchingSelector(selector labels.Selector) []*v1.Pod { +func (env *Environment) ExpectPodsMatchingSelector(selector labels.Selector) []*corev1.Pod { GinkgoHelper() - podList := &v1.PodList{} + podList := &corev1.PodList{} Expect(env.Client.List(env.Context, podList, client.MatchingLabelsSelector{Selector: selector})).To(Succeed()) return lo.ToSlicePtr(podList.Items) } @@ -462,7 +496,7 @@ func (env *Environment) EventuallyExpectNotFoundAssertion(objects ...client.Obje }) } -func (env *Environment) ExpectCreatedNodeCount(comparator string, count int) []*v1.Node { +func (env *Environment) ExpectCreatedNodeCount(comparator string, count int) []*corev1.Node { GinkgoHelper() createdNodes := env.Monitor.CreatedNodes() Expect(len(createdNodes)).To(BeNumerically(comparator, count), @@ -470,38 +504,40 @@ func (env *Environment) ExpectCreatedNodeCount(comparator string, count int) []* return createdNodes } -func (env *Environment) ExpectNodeCount(comparator string, count int) { +func (env *Environment) ExpectNodeCount(comparator string, count int) []*corev1.Node { GinkgoHelper() - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) Expect(len(nodeList.Items)).To(BeNumerically(comparator, count)) + return lo.ToSlicePtr(nodeList.Items) } -func (env *Environment) ExpectNodeClaimCount(comparator string, count int) { +func (env *Environment) ExpectNodeClaimCount(comparator string, count int) []*karpv1.NodeClaim { GinkgoHelper() - nodeClaimList := &corev1beta1.NodeClaimList{} + nodeClaimList := &karpv1.NodeClaimList{} Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) Expect(len(nodeClaimList.Items)).To(BeNumerically(comparator, count)) + return lo.ToSlicePtr(nodeClaimList.Items) } -func NodeClaimNames(nodeClaims []*corev1beta1.NodeClaim) []string { - return lo.Map(nodeClaims, func(n *corev1beta1.NodeClaim, index int) string { +func NodeClaimNames(nodeClaims []*karpv1.NodeClaim) []string { + return lo.Map(nodeClaims, func(n *karpv1.NodeClaim, index int) string { return n.Name }) } -func NodeNames(nodes []*v1.Node) []string { - return lo.Map(nodes, func(n *v1.Node, index int) string { +func NodeNames(nodes []*corev1.Node) []string { + return lo.Map(nodes, func(n *corev1.Node, index int) string { return n.Name }) } -func (env *Environment) ConsistentlyExpectNodeCount(comparator string, count int, duration time.Duration) []*v1.Node { +func (env *Environment) ConsistentlyExpectNodeCount(comparator string, count int, duration time.Duration) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("expecting nodes to be %s to %d for %s", comparator, count, duration)) - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} Consistently(func(g Gomega) { g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), @@ -510,28 +546,28 @@ func (env *Environment) ConsistentlyExpectNodeCount(comparator string, count int return lo.ToSlicePtr(nodeList.Items) } -func (env *Environment) ConsistentlyExpectNoDisruptions(nodeCount int, duration time.Duration) (taintedNodes []*v1.Node) { +func (env *Environment) ConsistentlyExpectNoDisruptions(nodeCount int, duration time.Duration) (taintedNodes []*corev1.Node) { GinkgoHelper() return env.ConsistentlyExpectDisruptionsWithNodeCount(0, nodeCount, duration) } // ConsistentlyExpectDisruptionsWithNodeCount will continually ensure that there are exactly disruptingNodes with totalNodes (including replacements and existing nodes) -func (env *Environment) ConsistentlyExpectDisruptionsWithNodeCount(disruptingNodes, totalNodes int, duration time.Duration) (taintedNodes []*v1.Node) { +func (env *Environment) ConsistentlyExpectDisruptionsWithNodeCount(disruptingNodes, totalNodes int, duration time.Duration) (taintedNodes []*corev1.Node) { GinkgoHelper() - nodes := []v1.Node{} + nodes := []corev1.Node{} Consistently(func(g Gomega) { // Ensure we don't change our NodeClaims - nodeClaimList := &corev1beta1.NodeClaimList{} + nodeClaimList := &karpv1.NodeClaimList{} g.Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) g.Expect(nodeClaimList.Items).To(HaveLen(totalNodes)) - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) g.Expect(nodeList.Items).To(HaveLen(totalNodes)) - nodes = lo.Filter(nodeList.Items, func(n v1.Node, _ int) bool { - _, ok := lo.Find(n.Spec.Taints, func(t v1.Taint) bool { - return corev1beta1.IsDisruptingTaint(t) + nodes = lo.Filter(nodeList.Items, func(n corev1.Node, _ int) bool { + _, ok := lo.Find(n.Spec.Taints, func(t corev1.Taint) bool { + return karpv1.IsDisruptingTaint(t) }) return ok }) @@ -540,33 +576,33 @@ func (env *Environment) ConsistentlyExpectDisruptionsWithNodeCount(disruptingNod return lo.ToSlicePtr(nodes) } -func (env *Environment) EventuallyExpectTaintedNodeCount(comparator string, count int) []*v1.Node { +func (env *Environment) EventuallyExpectTaintedNodeCount(comparator string, count int) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for tainted nodes to be %s to %d", comparator, count)) - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} Eventually(func(g Gomega) { - g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.taints[*].karpenter.sh/disruption": "disrupting"})).To(Succeed()) + g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.taints[*].karpenter.sh/disrupted": "true"})).To(Succeed()) g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), fmt.Sprintf("expected %d tainted nodes, had %d (%v)", count, len(nodeList.Items), NodeNames(lo.ToSlicePtr(nodeList.Items)))) }).Should(Succeed()) return lo.ToSlicePtr(nodeList.Items) } -func (env *Environment) EventuallyExpectNodesUntaintedWithTimeout(timeout time.Duration, nodes ...*v1.Node) { +func (env *Environment) EventuallyExpectNodesUntaintedWithTimeout(timeout time.Duration, nodes ...*corev1.Node) { GinkgoHelper() By(fmt.Sprintf("waiting for %d nodes to be untainted", len(nodes))) - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} Eventually(func(g Gomega) { - g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.taints[*].karpenter.sh/disruption": "disrupting"})).To(Succeed()) - taintedNodeNames := lo.Map(nodeList.Items, func(n v1.Node, _ int) string { return n.Name }) - g.Expect(taintedNodeNames).ToNot(ContainElements(lo.Map(nodes, func(n *v1.Node, _ int) interface{} { return n.Name })...)) + g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.taints[*].karpenter.sh/disrupted": "true"})).To(Succeed()) + taintedNodeNames := lo.Map(nodeList.Items, func(n corev1.Node, _ int) string { return n.Name }) + g.Expect(taintedNodeNames).ToNot(ContainElements(lo.Map(nodes, func(n *corev1.Node, _ int) interface{} { return n.Name })...)) }).WithTimeout(timeout).Should(Succeed()) } -func (env *Environment) EventuallyExpectNodeClaimCount(comparator string, count int) []*corev1beta1.NodeClaim { +func (env *Environment) EventuallyExpectNodeClaimCount(comparator string, count int) []*karpv1.NodeClaim { GinkgoHelper() By(fmt.Sprintf("waiting for nodes to be %s to %d", comparator, count)) - nodeClaimList := &corev1beta1.NodeClaimList{} + nodeClaimList := &karpv1.NodeClaimList{} Eventually(func(g Gomega) { g.Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) g.Expect(len(nodeClaimList.Items)).To(BeNumerically(comparator, count), @@ -575,10 +611,10 @@ func (env *Environment) EventuallyExpectNodeClaimCount(comparator string, count return lo.ToSlicePtr(nodeClaimList.Items) } -func (env *Environment) EventuallyExpectNodeCount(comparator string, count int) []*v1.Node { +func (env *Environment) EventuallyExpectNodeCount(comparator string, count int) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for nodes to be %s to %d", comparator, count)) - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} Eventually(func(g Gomega) { g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), @@ -587,10 +623,10 @@ func (env *Environment) EventuallyExpectNodeCount(comparator string, count int) return lo.ToSlicePtr(nodeList.Items) } -func (env *Environment) EventuallyExpectNodeCountWithSelector(comparator string, count int, selector labels.Selector) []*v1.Node { +func (env *Environment) EventuallyExpectNodeCountWithSelector(comparator string, count int, selector labels.Selector) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for nodes with selector %v to be %s to %d", selector, comparator, count)) - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} Eventually(func(g Gomega) { g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel}, client.MatchingLabelsSelector{Selector: selector})).To(Succeed()) g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), @@ -599,10 +635,10 @@ func (env *Environment) EventuallyExpectNodeCountWithSelector(comparator string, return lo.ToSlicePtr(nodeList.Items) } -func (env *Environment) EventuallyExpectCreatedNodeCount(comparator string, count int) []*v1.Node { +func (env *Environment) EventuallyExpectCreatedNodeCount(comparator string, count int) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for created nodes to be %s to %d", comparator, count)) - var createdNodes []*v1.Node + var createdNodes []*corev1.Node Eventually(func(g Gomega) { createdNodes = env.Monitor.CreatedNodes() g.Expect(len(createdNodes)).To(BeNumerically(comparator, count), @@ -611,10 +647,10 @@ func (env *Environment) EventuallyExpectCreatedNodeCount(comparator string, coun return createdNodes } -func (env *Environment) EventuallyExpectDeletedNodeCount(comparator string, count int) []*v1.Node { +func (env *Environment) EventuallyExpectDeletedNodeCount(comparator string, count int) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for deleted nodes to be %s to %d", comparator, count)) - var deletedNodes []*v1.Node + var deletedNodes []*corev1.Node Eventually(func(g Gomega) { deletedNodes = env.Monitor.DeletedNodes() g.Expect(len(deletedNodes)).To(BeNumerically(comparator, count), @@ -623,13 +659,13 @@ func (env *Environment) EventuallyExpectDeletedNodeCount(comparator string, coun return deletedNodes } -func (env *Environment) EventuallyExpectDeletedNodeCountWithSelector(comparator string, count int, selector labels.Selector) []*v1.Node { +func (env *Environment) EventuallyExpectDeletedNodeCountWithSelector(comparator string, count int, selector labels.Selector) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for deleted nodes with selector %v to be %s to %d", selector, comparator, count)) - var deletedNodes []*v1.Node + var deletedNodes []*corev1.Node Eventually(func(g Gomega) { deletedNodes = env.Monitor.DeletedNodes() - deletedNodes = lo.Filter(deletedNodes, func(n *v1.Node, _ int) bool { + deletedNodes = lo.Filter(deletedNodes, func(n *corev1.Node, _ int) bool { return selector.Matches(labels.Set(n.Labels)) }) g.Expect(len(deletedNodes)).To(BeNumerically(comparator, count), @@ -638,89 +674,79 @@ func (env *Environment) EventuallyExpectDeletedNodeCountWithSelector(comparator return deletedNodes } -func (env *Environment) EventuallyExpectInitializedNodeCount(comparator string, count int) []*v1.Node { +func (env *Environment) EventuallyExpectInitializedNodeCount(comparator string, count int) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for initialized nodes to be %s to %d", comparator, count)) - var nodes []*v1.Node + var nodes []*corev1.Node Eventually(func(g Gomega) { nodes = env.Monitor.CreatedNodes() - nodes = lo.Filter(nodes, func(n *v1.Node, _ int) bool { - return n.Labels[corev1beta1.NodeInitializedLabelKey] == "true" + nodes = lo.Filter(nodes, func(n *corev1.Node, _ int) bool { + return n.Labels[karpv1.NodeInitializedLabelKey] == "true" }) g.Expect(len(nodes)).To(BeNumerically(comparator, count)) }).Should(Succeed()) return nodes } -func (env *Environment) EventuallyExpectCreatedNodeClaimCount(comparator string, count int) []*corev1beta1.NodeClaim { +func (env *Environment) EventuallyExpectCreatedNodeClaimCount(comparator string, count int) []*karpv1.NodeClaim { GinkgoHelper() By(fmt.Sprintf("waiting for created nodeclaims to be %s to %d", comparator, count)) - nodeClaimList := &corev1beta1.NodeClaimList{} + nodeClaimList := &karpv1.NodeClaimList{} Eventually(func(g Gomega) { g.Expect(env.Client.List(env.Context, nodeClaimList)).To(Succeed()) g.Expect(len(nodeClaimList.Items)).To(BeNumerically(comparator, count)) }).Should(Succeed()) - return lo.Map(nodeClaimList.Items, func(nc corev1beta1.NodeClaim, _ int) *corev1beta1.NodeClaim { + return lo.Map(nodeClaimList.Items, func(nc karpv1.NodeClaim, _ int) *karpv1.NodeClaim { return &nc }) } -func (env *Environment) EventuallyExpectNodeClaimsReady(nodeClaims ...*corev1beta1.NodeClaim) { +func (env *Environment) EventuallyExpectNodeClaimsReady(nodeClaims ...*karpv1.NodeClaim) { GinkgoHelper() Eventually(func(g Gomega) { for _, nc := range nodeClaims { - temp := &corev1beta1.NodeClaim{} + temp := &karpv1.NodeClaim{} g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nc), temp)).Should(Succeed()) - g.Expect(temp.StatusConditions().IsHappy()).To(BeTrue()) - } - }).Should(Succeed()) -} - -func (env *Environment) EventuallyExpectExpired(nodeClaims ...*corev1beta1.NodeClaim) { - GinkgoHelper() - Eventually(func(g Gomega) { - for _, nc := range nodeClaims { - g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed()) - g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Expired).IsTrue()).To(BeTrue()) + g.Expect(temp.StatusConditions().Root().IsTrue()).To(BeTrue()) } }).Should(Succeed()) } -func (env *Environment) EventuallyExpectDrifted(nodeClaims ...*corev1beta1.NodeClaim) { +func (env *Environment) EventuallyExpectDrifted(nodeClaims ...*karpv1.NodeClaim) { GinkgoHelper() Eventually(func(g Gomega) { for _, nc := range nodeClaims { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed()) - g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Drifted).IsTrue()).To(BeTrue()) + g.Expect(nc.StatusConditions().Get(karpv1.ConditionTypeDrifted).IsTrue()).To(BeTrue()) } }).Should(Succeed()) } -func (env *Environment) ConsistentlyExpectNodeClaimsNotDrifted(duration time.Duration, nodeClaims ...*corev1beta1.NodeClaim) { +func (env *Environment) ConsistentlyExpectNodeClaimsNotDrifted(duration time.Duration, nodeClaims ...*karpv1.NodeClaim) { GinkgoHelper() - nodeClaimNames := lo.Map(nodeClaims, func(nc *corev1beta1.NodeClaim, _ int) string { return nc.Name }) + nodeClaimNames := lo.Map(nodeClaims, func(nc *karpv1.NodeClaim, _ int) string { return nc.Name }) By(fmt.Sprintf("consistently expect nodeclaims %s not to be drifted for %s", nodeClaimNames, duration)) Consistently(func(g Gomega) { for _, nc := range nodeClaims { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed()) - g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Drifted)).To(BeNil()) + g.Expect(nc.StatusConditions().Get(karpv1.ConditionTypeDrifted)).To(BeNil()) } }, duration).Should(Succeed()) } -func (env *Environment) EventuallyExpectEmpty(nodeClaims ...*corev1beta1.NodeClaim) { +func (env *Environment) EventuallyExpectConsolidatable(nodeClaims ...*karpv1.NodeClaim) { GinkgoHelper() Eventually(func(g Gomega) { for _, nc := range nodeClaims { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed()) - g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Empty).IsTrue()).To(BeTrue()) + g.Expect(nc.StatusConditions().Get(karpv1.ConditionTypeConsolidatable).IsTrue()).To(BeTrue()) } }).Should(Succeed()) } -func (env *Environment) GetNode(nodeName string) v1.Node { +func (env *Environment) GetNode(nodeName string) corev1.Node { GinkgoHelper() - var node v1.Node + var node corev1.Node Expect(env.Client.Get(env.Context, types.NamespacedName{Name: nodeName}, &node)).To(Succeed()) return node } @@ -738,7 +764,7 @@ var ( lastLogged = metav1.Now() ) -func (env *Environment) printControllerLogs(options *v1.PodLogOptions) { +func (env *Environment) printControllerLogs(options *corev1.PodLogOptions) { fmt.Println("------- START CONTROLLER LOGS -------") defer fmt.Println("------- END CONTROLLER LOGS -------") @@ -757,28 +783,28 @@ func (env *Environment) printControllerLogs(options *v1.PodLogOptions) { } stream, err := env.KubeClient.CoreV1().Pods("kube-system").GetLogs(pod.Name, temp).Stream(env.Context) if err != nil { - logging.FromContext(env.Context).Errorf("fetching controller logs: %s", err) + log.FromContext(env.Context).Error(err, "failed fetching controller logs") return } - log := &bytes.Buffer{} - _, err = io.Copy(log, stream) + raw := &bytes.Buffer{} + _, err = io.Copy(raw, stream) Expect(err).ToNot(HaveOccurred()) - logging.FromContext(env.Context).Info(log) + log.FromContext(env.Context).Info(raw.String()) } } -func (env *Environment) EventuallyExpectMinUtilization(resource v1.ResourceName, comparator string, value float64) { +func (env *Environment) EventuallyExpectMinUtilization(resource corev1.ResourceName, comparator string, value float64) { GinkgoHelper() Eventually(func(g Gomega) { g.Expect(env.Monitor.MinUtilization(resource)).To(BeNumerically(comparator, value)) }).Should(Succeed()) } -func (env *Environment) EventuallyExpectAvgUtilization(resource v1.ResourceName, comparator string, value float64) { +func (env *Environment) EventuallyExpectAvgUtilization(resource corev1.ResourceName, comparator string, value float64) { GinkgoHelper() Eventually(func(g Gomega) { g.Expect(env.Monitor.AvgUtilization(resource)).To(BeNumerically(comparator, value)) - }, 10*time.Minute).Should(Succeed()) + }, 12*time.Minute).Should(Succeed()) } func (env *Environment) ExpectDaemonSetEnvironmentVariableUpdated(obj client.ObjectKey, name, value string, containers ...string) { @@ -797,12 +823,12 @@ func (env *Environment) ExpectDaemonSetEnvironmentVariableUpdated(obj client.Obj continue } // If the env var already exists, update its value. Otherwise, create a new var. - if _, i, ok := lo.FindIndexOf(c.Env, func(e v1.EnvVar) bool { + if _, i, ok := lo.FindIndexOf(c.Env, func(e corev1.EnvVar) bool { return e.Name == name }); ok { c.Env[i].Value = value } else { - c.Env = append(c.Env, v1.EnvVar{Name: name, Value: value}) + c.Env = append(c.Env, corev1.EnvVar{Name: name, Value: value}) } } Expect(env.Client.Patch(env.Context, ds, patch)).To(Succeed()) @@ -811,7 +837,7 @@ func (env *Environment) ExpectDaemonSetEnvironmentVariableUpdated(obj client.Obj // ForcePodsToSpread ensures that currently scheduled pods get spread evenly across all passed nodes by deleting pods off of existing // nodes and waiting them to reschedule. This is useful for scenarios where you want to force the nodes be underutilized // but you want to keep a consistent count of nodes rather than leaving around empty ones. -func (env *Environment) ForcePodsToSpread(nodes ...*v1.Node) { +func (env *Environment) ForcePodsToSpread(nodes ...*corev1.Node) { GinkgoHelper() // Get the total count of pods across @@ -824,8 +850,8 @@ func (env *Environment) ForcePodsToSpread(nodes ...*v1.Node) { By(fmt.Sprintf("forcing %d pods to spread across %d nodes", podCount, len(nodes))) start := time.Now() for { - var nodePods []*v1.Pod - node, found := lo.Find(nodes, func(n *v1.Node) bool { + var nodePods []*corev1.Pod + node, found := lo.Find(nodes, func(n *corev1.Node) bool { nodePods = env.ExpectActivePodsForNode(n.Name) return len(nodePods) > maxPodsPerNode }) @@ -858,12 +884,12 @@ func (env *Environment) ForcePodsToSpread(nodes ...*v1.Node) { } } -func (env *Environment) ExpectActivePodsForNode(nodeName string) []*v1.Pod { +func (env *Environment) ExpectActivePodsForNode(nodeName string) []*corev1.Pod { GinkgoHelper() - podList := &v1.PodList{} + podList := &corev1.PodList{} Expect(env.Client.List(env, podList, client.MatchingFields{"spec.nodeName": nodeName}, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) - return lo.Filter(lo.ToSlicePtr(podList.Items), func(p *v1.Pod, _ int) bool { + return lo.Filter(lo.ToSlicePtr(podList.Items), func(p *corev1.Pod, _ int) bool { return p.DeletionTimestamp.IsZero() }) } @@ -878,11 +904,11 @@ func (env *Environment) ExpectCABundle() string { Expect(err).ToNot(HaveOccurred()) _, err = transport.TLSConfigFor(transportConfig) // fills in CAData! Expect(err).ToNot(HaveOccurred()) - logging.FromContext(env.Context).Debugf("Discovered caBundle, length %d", len(transportConfig.TLS.CAData)) + log.FromContext(env.Context).WithValues("length", len(transportConfig.TLS.CAData)).V(1).Info("discovered caBundle") return base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData) } -func (env *Environment) GetDaemonSetCount(np *corev1beta1.NodePool) int { +func (env *Environment) GetDaemonSetCount(np *karpv1.NodePool) int { GinkgoHelper() // Performs the same logic as the scheduler to get the number of daemonset @@ -891,7 +917,7 @@ func (env *Environment) GetDaemonSetCount(np *corev1beta1.NodePool) int { Expect(env.Client.List(env.Context, daemonSetList)).To(Succeed()) return lo.CountBy(daemonSetList.Items, func(d appsv1.DaemonSet) bool { - p := &v1.Pod{Spec: d.Spec.Template.Spec} + p := &corev1.Pod{Spec: d.Spec.Template.Spec} nodeClaimTemplate := pscheduling.NewNodeClaimTemplate(np) if err := scheduling.Taints(nodeClaimTemplate.Spec.Taints).Tolerates(p); err != nil { return false @@ -902,3 +928,24 @@ func (env *Environment) GetDaemonSetCount(np *corev1beta1.NodePool) int { return true }) } + +func (env *Environment) GetDaemonSetOverhead(np *karpv1.NodePool) corev1.ResourceList { + GinkgoHelper() + + // Performs the same logic as the scheduler to get the number of daemonset + // pods that we estimate we will need to schedule as overhead to each node + daemonSetList := &appsv1.DaemonSetList{} + Expect(env.Client.List(env.Context, daemonSetList)).To(Succeed()) + + return coreresources.RequestsForPods(lo.FilterMap(daemonSetList.Items, func(ds appsv1.DaemonSet, _ int) (*corev1.Pod, bool) { + p := &corev1.Pod{Spec: ds.Spec.Template.Spec} + nodeClaimTemplate := pscheduling.NewNodeClaimTemplate(np) + if err := scheduling.Taints(nodeClaimTemplate.Spec.Taints).Tolerates(p); err != nil { + return nil, false + } + if err := nodeClaimTemplate.Requirements.Compatible(scheduling.NewPodRequirements(p), scheduling.AllowUndefinedWellKnownLabels); err != nil { + return nil, false + } + return p, true + })...) +} diff --git a/test/pkg/environment/common/monitor.go b/test/pkg/environment/common/monitor.go index ab92f2c91ccf..bf43ce71599b 100644 --- a/test/pkg/environment/common/monitor.go +++ b/test/pkg/environment/common/monitor.go @@ -20,15 +20,15 @@ import ( "math" "sync" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" - "knative.dev/pkg/logging" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" "github.com/samber/lo" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/utils/resources" ) @@ -39,21 +39,21 @@ type Monitor struct { mu sync.RWMutex - nodesAtReset map[string]*v1.Node + nodesAtReset map[string]*corev1.Node } type state struct { - pods v1.PodList - nodes map[string]*v1.Node // node name -> node - nodePods map[string][]*v1.Pod // node name -> pods bound to the node - nodeRequests map[string]v1.ResourceList // node name -> sum of pod resource requests + pods corev1.PodList + nodes map[string]*corev1.Node // node name -> node + nodePods map[string][]*corev1.Pod // node name -> pods bound to the node + nodeRequests map[string]corev1.ResourceList // node name -> sum of pod resource requests } func NewMonitor(ctx context.Context, kubeClient client.Client) *Monitor { m := &Monitor{ ctx: ctx, kubeClient: kubeClient, - nodesAtReset: map[string]*v1.Node{}, + nodesAtReset: map[string]*corev1.Node{}, } m.Reset() return m @@ -105,36 +105,36 @@ func (m *Monitor) CreatedNodeCount() int { } // NodesAtReset returns a slice of nodes that the monitor saw at the last reset -func (m *Monitor) NodesAtReset() []*v1.Node { +func (m *Monitor) NodesAtReset() []*corev1.Node { m.mu.RLock() defer m.mu.RUnlock() return deepCopySlice(lo.Values(m.nodesAtReset)) } // Nodes returns all the nodes on the cluster -func (m *Monitor) Nodes() []*v1.Node { +func (m *Monitor) Nodes() []*corev1.Node { st := m.poll() return lo.Values(st.nodes) } // CreatedNodes returns the nodes that have been created since the last reset (essentially Nodes - NodesAtReset) -func (m *Monitor) CreatedNodes() []*v1.Node { - resetNodeNames := sets.NewString(lo.Map(m.NodesAtReset(), func(n *v1.Node, _ int) string { return n.Name })...) - return lo.Filter(m.Nodes(), func(n *v1.Node, _ int) bool { return !resetNodeNames.Has(n.Name) }) +func (m *Monitor) CreatedNodes() []*corev1.Node { + resetNodeNames := sets.NewString(lo.Map(m.NodesAtReset(), func(n *corev1.Node, _ int) string { return n.Name })...) + return lo.Filter(m.Nodes(), func(n *corev1.Node, _ int) bool { return !resetNodeNames.Has(n.Name) }) } // DeletedNodes returns the nodes that have been deleted since the last reset (essentially NodesAtReset - Nodes) -func (m *Monitor) DeletedNodes() []*v1.Node { - currentNodeNames := sets.NewString(lo.Map(m.Nodes(), func(n *v1.Node, _ int) string { return n.Name })...) - return lo.Filter(m.NodesAtReset(), func(n *v1.Node, _ int) bool { return !currentNodeNames.Has(n.Name) }) +func (m *Monitor) DeletedNodes() []*corev1.Node { + currentNodeNames := sets.NewString(lo.Map(m.Nodes(), func(n *corev1.Node, _ int) string { return n.Name })...) + return lo.Filter(m.NodesAtReset(), func(n *corev1.Node, _ int) bool { return !currentNodeNames.Has(n.Name) }) } // PendingPods returns the number of pending pods matching the given selector -func (m *Monitor) PendingPods(selector labels.Selector) []*v1.Pod { - var pods []*v1.Pod +func (m *Monitor) PendingPods(selector labels.Selector) []*corev1.Pod { + var pods []*corev1.Pod for _, pod := range m.poll().pods.Items { pod := pod - if pod.Status.Phase != v1.PodPending { + if pod.Status.Phase != corev1.PodPending { continue } if selector.Matches(labels.Set(pod.Labels)) { @@ -149,11 +149,11 @@ func (m *Monitor) PendingPodsCount(selector labels.Selector) int { } // RunningPods returns the number of running pods matching the given selector -func (m *Monitor) RunningPods(selector labels.Selector) []*v1.Pod { - var pods []*v1.Pod +func (m *Monitor) RunningPods(selector labels.Selector) []*corev1.Pod { + var pods []*corev1.Pod for _, pod := range m.poll().pods.Items { pod := pod - if pod.Status.Phase != v1.PodRunning { + if pod.Status.Phase != corev1.PodRunning { continue } if selector.Matches(labels.Set(pod.Labels)) { @@ -168,19 +168,19 @@ func (m *Monitor) RunningPodsCount(selector labels.Selector) int { } func (m *Monitor) poll() state { - var nodes v1.NodeList + var nodes corev1.NodeList if err := m.kubeClient.List(m.ctx, &nodes); err != nil { - logging.FromContext(m.ctx).Errorf("listing nodes, %s", err) + log.FromContext(m.ctx).Error(err, "failed listing nodes") } - var pods v1.PodList + var pods corev1.PodList if err := m.kubeClient.List(m.ctx, &pods); err != nil { - logging.FromContext(m.ctx).Errorf("listing pods, %s", err) + log.FromContext(m.ctx).Error(err, "failing listing pods") } st := state{ - nodes: map[string]*v1.Node{}, + nodes: map[string]*corev1.Node{}, pods: pods, - nodePods: map[string][]*v1.Pod{}, - nodeRequests: map[string]v1.ResourceList{}, + nodePods: map[string][]*corev1.Pod{}, + nodeRequests: map[string]corev1.ResourceList{}, } for i := range nodes.Items { st.nodes[nodes.Items[i].Name] = &nodes.Items[i] @@ -200,7 +200,7 @@ func (m *Monitor) poll() state { return st } -func (m *Monitor) AvgUtilization(resource v1.ResourceName) float64 { +func (m *Monitor) AvgUtilization(resource corev1.ResourceName) float64 { utilization := m.nodeUtilization(resource) sum := 0.0 for _, v := range utilization { @@ -209,7 +209,7 @@ func (m *Monitor) AvgUtilization(resource v1.ResourceName) float64 { return sum / float64(len(utilization)) } -func (m *Monitor) MinUtilization(resource v1.ResourceName) float64 { +func (m *Monitor) MinUtilization(resource corev1.ResourceName) float64 { min := math.MaxFloat64 for _, v := range m.nodeUtilization(resource) { min = math.Min(v, min) @@ -217,13 +217,13 @@ func (m *Monitor) MinUtilization(resource v1.ResourceName) float64 { return min } -func (m *Monitor) nodeUtilization(resource v1.ResourceName) []float64 { +func (m *Monitor) nodeUtilization(resource corev1.ResourceName) []float64 { st := m.poll() var utilization []float64 for nodeName, requests := range st.nodeRequests { allocatable := st.nodes[nodeName].Status.Allocatable[resource] // skip any nodes we didn't launch - if st.nodes[nodeName].Labels[v1beta1.NodePoolLabelKey] == "" { + if st.nodes[nodeName].Labels[karpv1.NodePoolLabelKey] == "" { continue } if allocatable.IsZero() { diff --git a/test/pkg/environment/common/setup.go b/test/pkg/environment/common/setup.go index cc233cc561ce..badee033ead6 100644 --- a/test/pkg/environment/common/setup.go +++ b/test/pkg/environment/common/setup.go @@ -21,7 +21,7 @@ import ( "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" schedulingv1 "k8s.io/api/scheduling/v1" storagev1 "k8s.io/api/storage/v1" @@ -31,11 +31,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" "sigs.k8s.io/karpenter/pkg/utils/pod" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/debug" . "github.com/onsi/ginkgo/v2" @@ -46,18 +46,20 @@ const TestingFinalizer = "testing/finalizer" var ( CleanableObjects = []client.Object{ - &v1.Pod{}, + &corev1.Pod{}, &appsv1.Deployment{}, + &appsv1.StatefulSet{}, &appsv1.DaemonSet{}, &policyv1.PodDisruptionBudget{}, - &v1.PersistentVolumeClaim{}, - &v1.PersistentVolume{}, + &corev1.PersistentVolumeClaim{}, + &corev1.PersistentVolume{}, &storagev1.StorageClass{}, - &corev1beta1.NodePool{}, - &v1.LimitRange{}, + &karpv1.NodePool{}, + &corev1.LimitRange{}, &schedulingv1.PriorityClass{}, - &v1.Node{}, - &corev1beta1.NodeClaim{}, + &corev1.Node{}, + &karpv1.NodeClaim{}, + &v1.EC2NodeClass{}, } ) @@ -73,14 +75,14 @@ func (env *Environment) BeforeEach() { } func (env *Environment) ExpectCleanCluster() { - var nodes v1.NodeList + var nodes corev1.NodeList Expect(env.Client.List(env.Context, &nodes)).To(Succeed()) for _, node := range nodes.Items { if len(node.Spec.Taints) == 0 && !node.Spec.Unschedulable { Fail(fmt.Sprintf("expected system pool node %s to be tainted", node.Name)) } } - var pods v1.PodList + var pods corev1.PodList Expect(env.Client.List(env.Context, &pods)).To(Succeed()) for i := range pods.Items { Expect(pod.IsProvisionable(&pods.Items[i])).To(BeFalse(), @@ -88,7 +90,7 @@ func (env *Environment) ExpectCleanCluster() { Expect(pods.Items[i].Namespace).ToNot(Equal("default"), fmt.Sprintf("expected no pods in the `default` namespace, found %s/%s", pods.Items[i].Namespace, pods.Items[i].Name)) } - for _, obj := range []client.Object{&corev1beta1.NodePool{}, &v1beta1.EC2NodeClass{}} { + for _, obj := range []client.Object{&karpv1.NodePool{}, &v1.EC2NodeClass{}} { metaList := &metav1.PartialObjectMetadataList{} gvk := lo.Must(apiutil.GVKForObject(obj, env.Client.Scheme())) metaList.SetGroupVersionKind(gvk) @@ -105,7 +107,7 @@ func (env *Environment) Cleanup() { func (env *Environment) AfterEach() { debug.AfterEach(env.Context) - env.printControllerLogs(&v1.PodLogOptions{Container: "controller"}) + env.printControllerLogs(&corev1.PodLogOptions{Container: "controller"}) } func (env *Environment) CleanupObjects(cleanableObjects ...client.Object) { diff --git a/test/suites/ami/suite_test.go b/test/suites/ami/suite_test.go new file mode 100644 index 000000000000..7ab83cb377c5 --- /dev/null +++ b/test/suites/ami/suite_test.go @@ -0,0 +1,357 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ami_test + +import ( + "encoding/base64" + "fmt" + "os" + "strings" + "testing" + "time" + + awssdk "github.com/aws/aws-sdk-go/aws" + + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/awslabs/operatorpkg/status" + . "github.com/awslabs/operatorpkg/test/expectations" + "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + environmentaws "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + + coretest "sigs.k8s.io/karpenter/pkg/test" +) + +var env *environmentaws.Environment +var nodeClass *v1.EC2NodeClass +var nodePool *karpv1.NodePool + +func TestAMI(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + env = environmentaws.NewEnvironment(t) + }) + AfterSuite(func() { + env.Stop() + }) + RunSpecs(t, "Ami") +} + +var _ = BeforeEach(func() { + env.BeforeEach() + nodeClass = env.DefaultEC2NodeClass() + nodePool = env.DefaultNodePool(nodeClass) +}) +var _ = AfterEach(func() { env.Cleanup() }) +var _ = AfterEach(func() { env.AfterEach() }) + +var _ = Describe("AMI", func() { + var customAMI string + BeforeEach(func() { + customAMI = env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersion())) + }) + + It("should use the AMI defined by the AMI Selector Terms", func() { + pod := coretest.Pod() + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: customAMI}} + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + + env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) + }) + It("should use the most recent AMI when discovering multiple", func() { + // choose an old static image that will definitely have an older creation date + oldCustomAMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%[1]s/amazon-linux-2023/x86_64/standard/amazon-eks-node-al2023-x86_64-standard-%[1]s-v20240514/image_id", env.K8sVersion())) + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + {ID: customAMI}, + {ID: oldCustomAMI}, + } + pod := coretest.Pod() + + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + + env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) + }) + It("should support AMI Selector Terms for Name but fail with incorrect owners", func() { + output, err := env.EC2API.DescribeImages(&ec2.DescribeImagesInput{ + ImageIds: []*string{awssdk.String(customAMI)}, + }) + Expect(err).To(BeNil()) + Expect(output.Images).To(HaveLen(1)) + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Name: *output.Images[0].Name, + Owner: "fakeOwnerValue", + }, + } + pod := coretest.Pod() + + env.ExpectCreated(pod, nodeClass, nodePool) + env.ExpectCreatedNodeCount("==", 0) + Expect(pod.Spec.NodeName).To(Equal("")) + }) + It("should support ami selector Name with default owners", func() { + output, err := env.EC2API.DescribeImages(&ec2.DescribeImagesInput{ + ImageIds: []*string{awssdk.String(customAMI)}, + }) + Expect(err).To(BeNil()) + Expect(output.Images).To(HaveLen(1)) + + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Name: *output.Images[0].Name, + }, + } + pod := coretest.Pod() + + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + + env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) + }) + It("should support ami selector ids", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + ID: customAMI, + }, + } + pod := coretest.Pod() + + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + + env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) + }) + + Context("AMIFamily", func() { + DescribeTable( + "should providion a node using an alias", + func(alias string) { + pod := coretest.Pod() + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: alias}} + env.ExpectCreated(nodeClass, nodePool, pod) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + }, + Entry("AL2023 (latest)", "al2023@latest"), + Entry("AL2023 (pinned)", "al2023@v20240807"), + Entry("AL2 (latest)", "al2@latest"), + Entry("AL2 (pinned)", "al2@v20240807"), + Entry("Bottlerocket (latest)", "bottlerocket@latest"), + Entry("Bottlerocket (pinned with v prefix)", "bottlerocket@v1.21.0"), + Entry("Bottlerocket (pinned without v prefix)", "bottlerocket@1.21.0"), + ) + It("should support Custom AMIFamily with AMI Selectors", func() { + al2023AMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersion())) + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: al2023AMI}} + rawContent, err := os.ReadFile("testdata/al2023_userdata_input.yaml") + Expect(err).ToNot(HaveOccurred()) + nodeClass.Spec.UserData = lo.ToPtr(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle())) + pod := coretest.Pod() + + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + + env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(al2023AMI)))) + }) + It("should have the EC2NodeClass status for AMIs using wildcard", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Name: "*", + }, + } + env.ExpectCreated(nodeClass) + nc := EventuallyExpectAMIsToExist(nodeClass) + Expect(len(nc.Status.AMIs)).To(BeNumerically("<", 10)) + }) + It("should have the EC2NodeClass status for AMIs using tags", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: customAMI}} + env.ExpectCreated(nodeClass) + nc := EventuallyExpectAMIsToExist(nodeClass) + Expect(len(nc.Status.AMIs)).To(BeNumerically("==", 1)) + Expect(nc.Status.AMIs[0].ID).To(Equal(customAMI)) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: v1.ConditionTypeAMIsReady, Status: metav1.ConditionTrue}) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: status.ConditionReady, Status: metav1.ConditionTrue}) + }) + It("should have ec2nodeClass status as not ready since AMI was not resolved", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: "ami-123"}} + env.ExpectCreated(nodeClass) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: v1.ConditionTypeAMIsReady, Status: metav1.ConditionFalse, Message: "AMISelector did not match any AMIs"}) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: status.ConditionReady, Status: metav1.ConditionFalse, Message: "AMIsReady=False"}) + }) + }) + + Context("UserData", func() { + It("should merge UserData contents for AL2 AMIFamily", func() { + content, err := os.ReadFile("testdata/al2_userdata_input.sh") + Expect(err).ToNot(HaveOccurred()) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} + nodeClass.Spec.UserData = awssdk.String(string(content)) + nodePool.Spec.Template.Spec.Taints = []corev1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} + nodePool.Spec.Template.Spec.StartupTaints = []corev1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} + pod := coretest.Pod(coretest.PodOptions{Tolerations: []corev1.Toleration{{Key: "example.com", Operator: corev1.TolerationOpExists}}}) + + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + Expect(env.GetNode(pod.Spec.NodeName).Spec.Taints).To(ContainElements( + corev1.Taint{Key: "example.com", Value: "value", Effect: "NoExecute"}, + corev1.Taint{Key: "example.com", Value: "value", Effect: "NoSchedule"}, + )) + actualUserData, err := base64.StdEncoding.DecodeString(*getInstanceAttribute(pod.Spec.NodeName, "userData").UserData.Value) + Expect(err).ToNot(HaveOccurred()) + // Since the node has joined the cluster, we know our bootstrapping was correct. + // Just verify if the UserData contains our custom content too, rather than doing a byte-wise comparison. + Expect(string(actualUserData)).To(ContainSubstring("Running custom user data script")) + }) + It("should merge non-MIME UserData contents for AL2 AMIFamily", func() { + content, err := os.ReadFile("testdata/al2_no_mime_userdata_input.sh") + Expect(err).ToNot(HaveOccurred()) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} + nodeClass.Spec.UserData = awssdk.String(string(content)) + nodePool.Spec.Template.Spec.Taints = []corev1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} + nodePool.Spec.Template.Spec.StartupTaints = []corev1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} + pod := coretest.Pod(coretest.PodOptions{Tolerations: []corev1.Toleration{{Key: "example.com", Operator: corev1.TolerationOpExists}}}) + + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + Expect(env.GetNode(pod.Spec.NodeName).Spec.Taints).To(ContainElements( + corev1.Taint{Key: "example.com", Value: "value", Effect: "NoExecute"}, + corev1.Taint{Key: "example.com", Value: "value", Effect: "NoSchedule"}, + )) + actualUserData, err := base64.StdEncoding.DecodeString(*getInstanceAttribute(pod.Spec.NodeName, "userData").UserData.Value) + Expect(err).ToNot(HaveOccurred()) + // Since the node has joined the cluster, we know our bootstrapping was correct. + // Just verify if the UserData contains our custom content too, rather than doing a byte-wise comparison. + Expect(string(actualUserData)).To(ContainSubstring("Running custom user data script")) + }) + It("should merge UserData contents for Bottlerocket AMIFamily", func() { + content, err := os.ReadFile("testdata/br_userdata_input.sh") + Expect(err).ToNot(HaveOccurred()) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} + nodeClass.Spec.UserData = awssdk.String(string(content)) + nodePool.Spec.Template.Spec.Taints = []corev1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} + nodePool.Spec.Template.Spec.StartupTaints = []corev1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} + pod := coretest.Pod(coretest.PodOptions{Tolerations: []corev1.Toleration{{Key: "example.com", Operator: corev1.TolerationOpExists}}}) + + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + Expect(env.GetNode(pod.Spec.NodeName).Spec.Taints).To(ContainElements( + corev1.Taint{Key: "example.com", Value: "value", Effect: "NoExecute"}, + corev1.Taint{Key: "example.com", Value: "value", Effect: "NoSchedule"}, + )) + actualUserData, err := base64.StdEncoding.DecodeString(*getInstanceAttribute(pod.Spec.NodeName, "userData").UserData.Value) + Expect(err).ToNot(HaveOccurred()) + Expect(string(actualUserData)).To(ContainSubstring("kube-api-qps = 30")) + }) + // Windows tests are can flake due to the instance types that are used in testing. + // The VPC Resource controller will need to support the instance types that are used. + // If the instance type is not supported by the controller resource `vpc.amazonaws.com/PrivateIPv4Address` will not register. + // Issue: https://github.com/aws/karpenter-provider-aws/issues/4472 + // See: https://github.com/aws/amazon-vpc-resource-controller-k8s/blob/master/pkg/aws/vpc/limits.go + It("should merge UserData contents for Windows AMIFamily", func() { + env.ExpectWindowsIPAMEnabled() + DeferCleanup(func() { + env.ExpectWindowsIPAMDisabled() + }) + + content, err := os.ReadFile("testdata/windows_userdata_input.ps1") + Expect(err).ToNot(HaveOccurred()) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "windows2022@latest"}} + nodeClass.Spec.UserData = awssdk.String(string(content)) + nodePool.Spec.Template.Spec.Taints = []corev1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} + nodePool.Spec.Template.Spec.StartupTaints = []corev1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} + + nodePool = coretest.ReplaceRequirements(nodePool, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Windows)}, + }, + }, + ) + pod := coretest.Pod(coretest.PodOptions{ + Image: environmentaws.WindowsDefaultImage, + NodeSelector: map[string]string{ + corev1.LabelOSStable: string(corev1.Windows), + corev1.LabelWindowsBuild: "10.0.20348", + }, + Tolerations: []corev1.Toleration{{Key: "example.com", Operator: corev1.TolerationOpExists}}, + }) + + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthyWithTimeout(time.Minute*15, pod) // Wait 15 minutes because Windows nodes/containers take longer to spin up + Expect(env.GetNode(pod.Spec.NodeName).Spec.Taints).To(ContainElements( + corev1.Taint{Key: "example.com", Value: "value", Effect: "NoExecute"}, + corev1.Taint{Key: "example.com", Value: "value", Effect: "NoSchedule"}, + )) + actualUserData, err := base64.StdEncoding.DecodeString(*getInstanceAttribute(pod.Spec.NodeName, "userData").UserData.Value) + Expect(err).ToNot(HaveOccurred()) + Expect(string(actualUserData)).To(ContainSubstring("Write-Host \"Running custom user data script\"")) + Expect(string(actualUserData)).To(ContainSubstring("[string]$EKSBootstrapScriptFile = \"$env:ProgramFiles\\Amazon\\EKS\\Start-EKSBootstrap.ps1\"")) + }) + }) +}) + +//nolint:unparam +func getInstanceAttribute(nodeName string, attribute string) *ec2.DescribeInstanceAttributeOutput { + var node corev1.Node + Expect(env.Client.Get(env.Context, types.NamespacedName{Name: nodeName}, &node)).To(Succeed()) + providerIDSplit := strings.Split(node.Spec.ProviderID, "/") + instanceID := providerIDSplit[len(providerIDSplit)-1] + instanceAttribute, err := env.EC2API.DescribeInstanceAttribute(&ec2.DescribeInstanceAttributeInput{ + InstanceId: awssdk.String(instanceID), + Attribute: awssdk.String(attribute), + }) + Expect(err).ToNot(HaveOccurred()) + return instanceAttribute +} + +func EventuallyExpectAMIsToExist(nodeClass *v1.EC2NodeClass) *v1.EC2NodeClass { + nc := &v1.EC2NodeClass{} + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClass), nc)).To(Succeed()) + g.Expect(nc.Status.AMIs).ToNot(BeNil()) + }).WithTimeout(30 * time.Second).Should(Succeed()) + return nc +} diff --git a/test/suites/ami/testdata/al2023_userdata_input.yaml b/test/suites/ami/testdata/al2023_userdata_input.yaml new file mode 100644 index 000000000000..b0ce7a5e8496 --- /dev/null +++ b/test/suites/ami/testdata/al2023_userdata_input.yaml @@ -0,0 +1,14 @@ +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: %s + apiServerEndpoint: %s + certificateAuthority: %s + cidr: 10.100.0.0/16 + kubelet: + config: + clusterDNS: + - 10.0.100.10 + flags: + - --node-labels="testing/cluster=unspecified" \ No newline at end of file diff --git a/test/suites/integration/testdata/al2_no_mime_userdata_input.sh b/test/suites/ami/testdata/al2_no_mime_userdata_input.sh similarity index 100% rename from test/suites/integration/testdata/al2_no_mime_userdata_input.sh rename to test/suites/ami/testdata/al2_no_mime_userdata_input.sh diff --git a/test/suites/integration/testdata/al2_userdata_input.sh b/test/suites/ami/testdata/al2_userdata_input.sh similarity index 100% rename from test/suites/integration/testdata/al2_userdata_input.sh rename to test/suites/ami/testdata/al2_userdata_input.sh diff --git a/test/suites/integration/testdata/br_userdata_input.sh b/test/suites/ami/testdata/br_userdata_input.sh similarity index 100% rename from test/suites/integration/testdata/br_userdata_input.sh rename to test/suites/ami/testdata/br_userdata_input.sh diff --git a/test/suites/integration/testdata/windows_userdata_input.ps1 b/test/suites/ami/testdata/windows_userdata_input.ps1 similarity index 100% rename from test/suites/integration/testdata/windows_userdata_input.ps1 rename to test/suites/ami/testdata/windows_userdata_input.ps1 diff --git a/test/suites/chaos/suite_test.go b/test/suites/chaos/suite_test.go index 9011876d9a36..8023a9cdb20c 100644 --- a/test/suites/chaos/suite_test.go +++ b/test/suites/chaos/suite_test.go @@ -22,7 +22,7 @@ import ( "time" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" "k8s.io/client-go/rest" @@ -33,11 +33,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coretest "sigs.k8s.io/karpenter/pkg/test" nodeutils "sigs.k8s.io/karpenter/pkg/utils/node" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/debug" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" @@ -46,8 +46,8 @@ import ( ) var env *aws.Environment -var nodeClass *v1beta1.EC2NodeClass -var nodePool *corev1beta1.NodePool +var nodeClass *v1.EC2NodeClass +var nodePool *karpv1.NodePool func TestChaos(t *testing.T) { RegisterFailHandler(Fail) @@ -74,15 +74,15 @@ var _ = Describe("Chaos", func() { ctx, cancel := context.WithCancel(env.Context) defer cancel() - nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, + nodePool = coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeSpot}, }, }) - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenUnderutilized - nodePool.Spec.Disruption.ConsolidateAfter = nil + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmptyOrUnderutilized + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") numPods := 1 dep := coretest.Deployment(coretest.DeploymentOptions{ @@ -103,7 +103,7 @@ var _ = Describe("Chaos", func() { // Expect that we never get over a high number of nodes Consistently(func(g Gomega) { - list := &v1.NodeList{} + list := &corev1.NodeList{} g.Expect(env.Client.List(env.Context, list, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) g.Expect(len(list.Items)).To(BeNumerically("<", 35)) }, time.Minute*5).Should(Succeed()) @@ -112,8 +112,8 @@ var _ = Describe("Chaos", func() { ctx, cancel := context.WithCancel(env.Context) defer cancel() - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenEmpty - nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(30 * time.Second)} + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmpty + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("30s") numPods := 1 dep := coretest.Deployment(coretest.DeploymentOptions{ Replicas: int32(numPods), @@ -133,7 +133,7 @@ var _ = Describe("Chaos", func() { // Expect that we never get over a high number of nodes Consistently(func(g Gomega) { - list := &v1.NodeList{} + list := &corev1.NodeList{} g.Expect(env.Client.List(env.Context, list, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) g.Expect(len(list.Items)).To(BeNumerically("<", 35)) }, time.Minute*5).Should(Succeed()) @@ -146,15 +146,15 @@ type taintAdder struct { } func (t *taintAdder) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - node := &v1.Node{} + node := &corev1.Node{} if err := t.kubeClient.Get(ctx, req.NamespacedName, node); err != nil { return reconcile.Result{}, client.IgnoreNotFound(err) } mergeFrom := client.StrategicMergeFrom(node.DeepCopy()) - taint := v1.Taint{ + taint := corev1.Taint{ Key: "test", Value: "true", - Effect: v1.TaintEffectNoExecute, + Effect: corev1.TaintEffectNoExecute, } if !lo.Contains(node.Spec.Taints, taint) { node.Spec.Taints = append(node.Spec.Taints, taint) @@ -167,9 +167,9 @@ func (t *taintAdder) Reconcile(ctx context.Context, req reconcile.Request) (reco func (t *taintAdder) Builder(mgr manager.Manager) *controllerruntime.Builder { return controllerruntime.NewControllerManagedBy(mgr). - For(&v1.Node{}). + For(&corev1.Node{}). WithEventFilter(predicate.NewPredicateFuncs(func(obj client.Object) bool { - node := obj.(*v1.Node) + node := obj.(*corev1.Node) if _, ok := node.Labels[coretest.DiscoveryLabel]; !ok { return false } @@ -197,7 +197,7 @@ func startNodeCountMonitor(ctx context.Context, kubeClient client.Client) { deletedNodes := atomic.Int64{} factory := informers.NewSharedInformerFactoryWithOptions(env.KubeClient, time.Second*30, - informers.WithTweakListOptions(func(l *metav1.ListOptions) { l.LabelSelector = corev1beta1.NodePoolLabelKey })) + informers.WithTweakListOptions(func(l *metav1.ListOptions) { l.LabelSelector = karpv1.NodePoolLabelKey })) nodeInformer := factory.Core().V1().Nodes().Informer() _ = lo.Must(nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(_ interface{}) { @@ -210,10 +210,10 @@ func startNodeCountMonitor(ctx context.Context, kubeClient client.Client) { factory.Start(ctx.Done()) go func() { for { - list := &v1.NodeList{} + list := &corev1.NodeList{} if err := kubeClient.List(ctx, list, client.HasLabels{coretest.DiscoveryLabel}); err == nil { - readyCount := lo.CountBy(list.Items, func(n v1.Node) bool { - return nodeutils.GetCondition(&n, v1.NodeReady).Status == v1.ConditionTrue + readyCount := lo.CountBy(list.Items, func(n corev1.Node) bool { + return nodeutils.GetCondition(&n, corev1.NodeReady).Status == corev1.ConditionTrue }) fmt.Printf("[NODE COUNT] CURRENT: %d | READY: %d | CREATED: %d | DELETED: %d\n", len(list.Items), readyCount, createdNodes.Load(), deletedNodes.Load()) } diff --git a/test/suites/consolidation/suite_test.go b/test/suites/consolidation/suite_test.go index 3ed5408c5aa0..6d6ebb652514 100644 --- a/test/suites/consolidation/suite_test.go +++ b/test/suites/consolidation/suite_test.go @@ -21,18 +21,20 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/awslabs/operatorpkg/object" "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/debug" environmentaws "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" @@ -55,7 +57,7 @@ func TestConsolidation(t *testing.T) { RunSpecs(t, "Consolidation") } -var nodeClass *v1beta1.EC2NodeClass +var nodeClass *v1.EC2NodeClass var _ = BeforeEach(func() { nodeClass = env.DefaultEC2NodeClass() @@ -65,14 +67,127 @@ var _ = AfterEach(func() { env.Cleanup() }) var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Consolidation", func() { + Context("LastPodEventTime", func() { + var nodePool *karpv1.NodePool + BeforeEach(func() { + nodePool = env.DefaultNodePool(nodeClass) + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("Never") + + }) + It("should update lastPodEventTime when pods are scheduled and removed", func() { + var numPods int32 = 5 + dep := test.Deployment(test.DeploymentOptions{ + Replicas: numPods, + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "regular-app"}, + }, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, + }, + }) + selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{ + { + Nodes: "0%", + }, + } + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 1) + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + + nodeClaim := env.ExpectExists(nodeClaims[0]).(*karpv1.NodeClaim) + lastPodEventTime := nodeClaim.Status.LastPodEventTime + + // wait 10 seconds so that we don't run into the de-dupe timeout + time.Sleep(10 * time.Second) + + dep.Spec.Replicas = lo.ToPtr[int32](4) + By("removing one pod from the node") + env.ExpectUpdated(dep) + + Eventually(func(g Gomega) { + nodeClaim = env.ExpectExists(nodeClaim).(*karpv1.NodeClaim) + g.Expect(nodeClaim.Status.LastPodEventTime.Time).ToNot(BeEquivalentTo(lastPodEventTime.Time)) + }).WithTimeout(5 * time.Second).WithPolling(1 * time.Second).Should(Succeed()) + lastPodEventTime = nodeClaim.Status.LastPodEventTime + + // wait 10 seconds so that we don't run into the de-dupe timeout + time.Sleep(10 * time.Second) + + dep.Spec.Replicas = lo.ToPtr[int32](5) + By("adding one pod to the node") + env.ExpectUpdated(dep) + + Eventually(func(g Gomega) { + nodeClaim = env.ExpectExists(nodeClaim).(*karpv1.NodeClaim) + g.Expect(nodeClaim.Status.LastPodEventTime.Time).ToNot(BeEquivalentTo(lastPodEventTime.Time)) + }).WithTimeout(5 * time.Second).WithPolling(1 * time.Second).Should(Succeed()) + }) + It("should update lastPodEventTime when pods go terminal", func() { + podLabels := map[string]string{"app": "regular-app"} + pod := test.Pod(test.PodOptions{ + // use a non-pause image so that we can have a sleep + Image: "alpine:3.20.2", + Command: []string{"/bin/sh", "-c", "sleep 30"}, + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, + RestartPolicy: corev1.RestartPolicyNever, + }) + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: test.RandomName(), + Namespace: "default", + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: pod.ObjectMeta, + Spec: pod.Spec, + }, + }, + } + selector := labels.SelectorFromSet(podLabels) + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{ + { + Nodes: "0%", + }, + } + env.ExpectCreated(nodeClass, nodePool, job) + + nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 1) + env.EventuallyExpectCreatedNodeCount("==", 1) + pods := env.EventuallyExpectHealthyPodCount(selector, int(1)) + + // pods are healthy, which means the job has started its 30s sleep + nodeClaim := env.ExpectExists(nodeClaims[0]).(*karpv1.NodeClaim) + lastPodEventTime := nodeClaim.Status.LastPodEventTime + + // wait a minute for the pod's sleep to finish, and for the nodeclaim to update + Eventually(func(g Gomega) { + pod := env.ExpectExists(pods[0]).(*corev1.Pod) + g.Expect(pod.Status.Phase).To(Equal(corev1.PodSucceeded)) + }).WithTimeout(1 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + + nodeClaim = env.ExpectExists(nodeClaims[0]).(*karpv1.NodeClaim) + Expect(nodeClaim.Status.LastPodEventTime).ToNot(BeEquivalentTo(lastPodEventTime.Time)) + }) + + }) Context("Budgets", func() { - var nodePool *corev1beta1.NodePool + var nodePool *karpv1.NodePool var dep *appsv1.Deployment var selector labels.Selector var numPods int32 BeforeEach(func() { nodePool = env.DefaultNodePool(nodeClass) - nodePool.Spec.Disruption.ConsolidateAfter = nil + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") numPods = 5 dep = test.Deployment(test.DeploymentOptions{ @@ -81,27 +196,27 @@ var _ = Describe("Consolidation", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "regular-app"}, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, }, }, }) selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) }) It("should respect budgets for empty delete consolidation", func() { - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{ + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{ { Nodes: "40%", }, } // Hostname anti-affinity to require one pod on each node - dep.Spec.Template.Spec.Affinity = &v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + dep.Spec.Template.Spec.Affinity = &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ { LabelSelector: dep.Spec.Selector, - TopologyKey: v1.LabelHostname, + TopologyKey: corev1.LabelHostname, }, }, }, @@ -124,7 +239,7 @@ var _ = Describe("Consolidation", func() { // Update the deployment to only contain 1 replica. env.ExpectUpdated(dep) - // Ensure that we get two nodes tainted, and they have overlap during the drift + // Ensure that we get two nodes tainted, and they have overlap during consolidation env.EventuallyExpectTaintedNodeCount("==", 2) nodes = env.ConsistentlyExpectDisruptionsWithNodeCount(2, 5, 5*time.Second) @@ -151,18 +266,18 @@ var _ = Describe("Consolidation", func() { }) It("should respect budgets for non-empty delete consolidation", func() { // This test will hold consolidation until we are ready to execute it - nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{} + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("Never") nodePool = test.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: v1.LabelInstanceSize, + Operator: corev1.NodeSelectorOpIn, Values: []string{"2xlarge"}, }, }, ) // We're expecting to create 3 nodes, so we'll expect to see at most 2 nodes deleting at one time. - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{{ Nodes: "50%", }} numPods = 9 @@ -173,9 +288,9 @@ var _ = Describe("Consolidation", func() { Labels: map[string]string{"app": "large-app"}, }, // Each 2xlarge has 8 cpu, so each node should fit no more than 3 pods. - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2100m"), + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2100m"), }, }, }, @@ -204,7 +319,7 @@ var _ = Describe("Consolidation", func() { } By("enabling consolidation") - nodePool.Spec.Disruption.ConsolidateAfter = nil + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") env.ExpectUpdated(nodePool) // Ensure that we get two nodes tainted, and they have overlap during consolidation @@ -220,27 +335,27 @@ var _ = Describe("Consolidation", func() { It("should respect budgets for non-empty replace consolidation", func() { appLabels := map[string]string{"app": "large-app"} // This test will hold consolidation until we are ready to execute it - nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{} + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("Never") nodePool = test.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceSize, + Operator: corev1.NodeSelectorOpIn, Values: []string{"xlarge", "2xlarge"}, }, }, // Add an Exists operator so that we can select on a fake partition later - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ Key: "test-partition", - Operator: v1.NodeSelectorOpExists, + Operator: corev1.NodeSelectorOpExists, }, }, ) nodePool.Labels = appLabels // We're expecting to create 5 nodes, so we'll expect to see at most 3 nodes deleting at one time. - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{{ Nodes: "3", }} @@ -252,9 +367,9 @@ var _ = Describe("Consolidation", func() { }, // Each 2xlarge has 8 cpu, so each node should fit no more than 1 pod since each node will have. // an equivalently sized daemonset - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("3"), + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), }, }, }, @@ -277,9 +392,9 @@ var _ = Describe("Consolidation", func() { NodeSelector: map[string]string{"test-partition": fmt.Sprintf("%d", i)}, // Each 2xlarge has 8 cpu, so each node should fit no more than 1 pod since each node will have. // an equivalently sized daemonset - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("3"), + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), }, }, }, @@ -308,7 +423,7 @@ var _ = Describe("Consolidation", func() { env.EventuallyExpectHealthyPodCount(selector, int(numPods)) By("enabling consolidation") - nodePool.Spec.Disruption.ConsolidateAfter = nil + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") env.ExpectUpdated(nodePool) // Ensure that we get three nodes tainted, and they have overlap during the consolidation @@ -324,24 +439,24 @@ var _ = Describe("Consolidation", func() { // Eventually expect all the nodes to be rolled and completely removed // Since this completes the disruption operation, this also ensures that we aren't leaking nodes into subsequent // tests since nodeclaims that are actively replacing but haven't brought-up nodes yet can register nodes later - env.EventuallyExpectNotFound(lo.Map(originalNodes, func(n *v1.Node, _ int) client.Object { return n })...) - env.EventuallyExpectNotFound(lo.Map(originalNodeClaims, func(n *corev1beta1.NodeClaim, _ int) client.Object { return n })...) + env.EventuallyExpectNotFound(lo.Map(originalNodes, func(n *corev1.Node, _ int) client.Object { return n })...) + env.EventuallyExpectNotFound(lo.Map(originalNodeClaims, func(n *karpv1.NodeClaim, _ int) client.Object { return n })...) env.ExpectNodeClaimCount("==", 5) env.ExpectNodeCount("==", 5) }) It("should not allow consolidation if the budget is fully blocking", func() { // We're going to define a budget that doesn't allow any consolidation to happen - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{{ Nodes: "0", }} // Hostname anti-affinity to require one pod on each node - dep.Spec.Template.Spec.Affinity = &v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + dep.Spec.Template.Spec.Affinity = &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ { LabelSelector: dep.Spec.Selector, - TopologyKey: v1.LabelHostname, + TopologyKey: corev1.LabelHostname, }, }, }, @@ -365,19 +480,19 @@ var _ = Describe("Consolidation", func() { // the current time and extends 15 minutes past the current time // Times need to be in UTC since the karpenter containers were built in UTC time windowStart := time.Now().Add(-time.Minute * 15).UTC() - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{{ Nodes: "0", Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())), Duration: &metav1.Duration{Duration: time.Minute * 30}, }} // Hostname anti-affinity to require one pod on each node - dep.Spec.Template.Spec.Affinity = &v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + dep.Spec.Template.Spec.Affinity = &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ { LabelSelector: dep.Spec.Selector, - TopologyKey: v1.LabelHostname, + TopologyKey: corev1.LabelHostname, }, }, }, @@ -398,34 +513,34 @@ var _ = Describe("Consolidation", func() { }) DescribeTable("should consolidate nodes (delete)", Label(debug.NoWatch), Label(debug.NoEvents), func(spotToSpot bool) { - nodePool := test.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Disruption: corev1beta1.Disruption{ - ConsolidationPolicy: corev1beta1.ConsolidationPolicyWhenUnderutilized, + nodePool := test.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Disruption: karpv1.Disruption{ + ConsolidationPolicy: karpv1.ConsolidationPolicyWhenEmptyOrUnderutilized, // Disable Consolidation until we're ready - ConsolidateAfter: &corev1beta1.NillableDuration{}, + ConsolidateAfter: karpv1.MustParseNillableDuration("Never"), }, - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: lo.Ternary(spotToSpot, []string{corev1beta1.CapacityTypeSpot}, []string{corev1beta1.CapacityTypeOnDemand}), + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: lo.Ternary(spotToSpot, []string{karpv1.CapacityTypeSpot}, []string{karpv1.CapacityTypeOnDemand}), }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceSize, + Operator: corev1.NodeSelectorOpIn, Values: []string{"medium", "large", "xlarge"}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceFamily, + Operator: corev1.NodeSelectorOpNotIn, // remove some cheap burstable and the odd c1 instance types so we have // more control over what gets provisioned // TODO: jmdeal@ remove a1 from exclusion list once Karpenter implicitly filters a1 instances for AL2023 AMI family (incompatible) @@ -433,7 +548,11 @@ var _ = Describe("Consolidation", func() { }, }, }, - NodeClassRef: &corev1beta1.NodeClassReference{Name: nodeClass.Name}, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, }, }, }, @@ -446,8 +565,8 @@ var _ = Describe("Consolidation", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "large-app"}, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, }, }, }) @@ -461,14 +580,14 @@ var _ = Describe("Consolidation", func() { // reduce the number of pods by 60% dep.Spec.Replicas = aws.Int32(40) env.ExpectUpdated(dep) - env.EventuallyExpectAvgUtilization(v1.ResourceCPU, "<", 0.5) + env.EventuallyExpectAvgUtilization(corev1.ResourceCPU, "<", 0.5) - // Enable consolidation as WhenUnderutilized doesn't allow a consolidateAfter value - nodePool.Spec.Disruption.ConsolidateAfter = nil + // Enable consolidation as WhenEmptyOrUnderutilized doesn't allow a consolidateAfter value + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") env.ExpectUpdated(nodePool) // With consolidation enabled, we now must delete nodes - env.EventuallyExpectAvgUtilization(v1.ResourceCPU, ">", 0.6) + env.EventuallyExpectAvgUtilization(corev1.ResourceCPU, ">", 0.6) env.ExpectDeleted(dep) }, @@ -477,41 +596,53 @@ var _ = Describe("Consolidation", func() { ) DescribeTable("should consolidate nodes (replace)", func(spotToSpot bool) { - nodePool := test.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Disruption: corev1beta1.Disruption{ - ConsolidationPolicy: corev1beta1.ConsolidationPolicyWhenUnderutilized, + nodePool := test.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Disruption: karpv1.Disruption{ + ConsolidationPolicy: karpv1.ConsolidationPolicyWhenEmptyOrUnderutilized, // Disable Consolidation until we're ready - ConsolidateAfter: &corev1beta1.NillableDuration{}, + ConsolidateAfter: karpv1.MustParseNillableDuration("Never"), }, - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: lo.Ternary(spotToSpot, []string{corev1beta1.CapacityTypeSpot}, []string{corev1beta1.CapacityTypeOnDemand}), + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: lo.Ternary(spotToSpot, []string{karpv1.CapacityTypeSpot}, []string{karpv1.CapacityTypeOnDemand}), }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceSize, + Operator: corev1.NodeSelectorOpIn, Values: []string{"large", "2xlarge"}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceFamily, + Operator: corev1.NodeSelectorOpNotIn, // remove some cheap burstable and the odd c1 / a1 instance types so we have // more control over what gets provisioned Values: []string{"t2", "t3", "c1", "t3a", "t4g", "a1"}, }, }, + // Specify Linux in the NodePool to filter out Windows only DS when discovering DS overhead + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Linux)}, + }, + }, + }, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, - NodeClassRef: &corev1beta1.NodeClassReference{Name: nodeClass.Name}, }, }, }, @@ -524,11 +655,11 @@ var _ = Describe("Consolidation", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "large-app"}, }, - TopologySpreadConstraints: []v1.TopologySpreadConstraint{ + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ { MaxSkew: 1, - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, + TopologyKey: corev1.LabelHostname, + WhenUnsatisfiable: corev1.DoNotSchedule, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": "large-app", @@ -536,8 +667,8 @@ var _ = Describe("Consolidation", func() { }, }, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, }, }, }) @@ -547,11 +678,11 @@ var _ = Describe("Consolidation", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "small-app"}, }, - TopologySpreadConstraints: []v1.TopologySpreadConstraint{ + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ { MaxSkew: 1, - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, + TopologyKey: corev1.LabelHostname, + WhenUnsatisfiable: corev1.DoNotSchedule, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": "small-app", @@ -559,8 +690,15 @@ var _ = Describe("Consolidation", func() { }, }, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1.5")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: func() resource.Quantity { + dsOverhead := env.GetDaemonSetOverhead(nodePool) + base := lo.ToPtr(resource.MustParse("1800m")) + base.Sub(*dsOverhead.Cpu()) + return *base + }(), + }, }, }, }) @@ -577,25 +715,25 @@ var _ = Describe("Consolidation", func() { // scaling down the large deployment leaves only small pods on each node largeDep.Spec.Replicas = aws.Int32(0) env.ExpectUpdated(largeDep) - env.EventuallyExpectAvgUtilization(v1.ResourceCPU, "<", 0.5) + env.EventuallyExpectAvgUtilization(corev1.ResourceCPU, "<", 0.5) - nodePool.Spec.Disruption.ConsolidateAfter = nil + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") env.ExpectUpdated(nodePool) // With consolidation enabled, we now must replace each node in turn to consolidate due to the anti-affinity // rules on the smaller deployment. The 2xl nodes should go to a large - env.EventuallyExpectAvgUtilization(v1.ResourceCPU, ">", 0.8) + env.EventuallyExpectAvgUtilization(corev1.ResourceCPU, ">", 0.8) - var nodes v1.NodeList + var nodes corev1.NodeList Expect(env.Client.List(env.Context, &nodes)).To(Succeed()) numLargeNodes := 0 numOtherNodes := 0 for _, n := range nodes.Items { // only count the nodes created by the provisoiner - if n.Labels[corev1beta1.NodePoolLabelKey] != nodePool.Name { + if n.Labels[karpv1.NodePoolLabelKey] != nodePool.Name { continue } - if strings.HasSuffix(n.Labels[v1.LabelInstanceTypeStable], ".large") { + if strings.HasSuffix(n.Labels[corev1.LabelInstanceTypeStable], ".large") { numLargeNodes++ } else { numOtherNodes++ @@ -613,41 +751,45 @@ var _ = Describe("Consolidation", func() { Entry("if the nodes are spot nodes", true), ) It("should consolidate on-demand nodes to spot (replace)", func() { - nodePool := test.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Disruption: corev1beta1.Disruption{ - ConsolidationPolicy: corev1beta1.ConsolidationPolicyWhenUnderutilized, + nodePool := test.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Disruption: karpv1.Disruption{ + ConsolidationPolicy: karpv1.ConsolidationPolicyWhenEmptyOrUnderutilized, // Disable Consolidation until we're ready - ConsolidateAfter: &corev1beta1.NillableDuration{}, + ConsolidateAfter: karpv1.MustParseNillableDuration("Never"), }, - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceSize, + Operator: corev1.NodeSelectorOpIn, Values: []string{"large"}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceFamily, + Operator: corev1.NodeSelectorOpNotIn, // remove some cheap burstable and the odd c1 / a1 instance types so we have // more control over what gets provisioned Values: []string{"t2", "t3", "c1", "t3a", "t4g", "a1"}, }, }, }, - NodeClassRef: &corev1beta1.NodeClassReference{Name: nodeClass.Name}, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, }, }, }, @@ -660,11 +802,11 @@ var _ = Describe("Consolidation", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "small-app"}, }, - TopologySpreadConstraints: []v1.TopologySpreadConstraint{ + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ { MaxSkew: 1, - TopologyKey: v1.LabelHostname, - WhenUnsatisfiable: v1.DoNotSchedule, + TopologyKey: corev1.LabelHostname, + WhenUnsatisfiable: corev1.DoNotSchedule, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": "small-app", @@ -672,8 +814,14 @@ var _ = Describe("Consolidation", func() { }, }, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1.5")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: func() resource.Quantity { + dsOverhead := env.GetDaemonSetOverhead(nodePool) + base := lo.ToPtr(resource.MustParse("1800m")) + base.Sub(*dsOverhead.Cpu()) + return *base + }(), + }, }, }, }) @@ -688,18 +836,18 @@ var _ = Describe("Consolidation", func() { // Enable spot capacity type after the on-demand node is provisioned // Expect the node to consolidate to a spot instance as it will be a cheaper // instance than on-demand - nodePool.Spec.Disruption.ConsolidateAfter = nil + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") test.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpExists, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpExists, }, }, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceSize, + Operator: corev1.NodeSelectorOpIn, Values: []string{"large"}, }, }, @@ -709,16 +857,16 @@ var _ = Describe("Consolidation", func() { // Eventually expect the on-demand nodes to be consolidated into // spot nodes after some time Eventually(func(g Gomega) { - var nodes v1.NodeList + var nodes corev1.NodeList Expect(env.Client.List(env.Context, &nodes)).To(Succeed()) - var spotNodes []*v1.Node - var otherNodes []*v1.Node + var spotNodes []*corev1.Node + var otherNodes []*corev1.Node for i, n := range nodes.Items { // only count the nodes created by the nodePool - if n.Labels[corev1beta1.NodePoolLabelKey] != nodePool.Name { + if n.Labels[karpv1.NodePoolLabelKey] != nodePool.Name { continue } - if n.Labels[corev1beta1.CapacityTypeLabelKey] == corev1beta1.CapacityTypeSpot { + if n.Labels[karpv1.CapacityTypeLabelKey] == karpv1.CapacityTypeSpot { spotNodes = append(spotNodes, &nodes.Items[i]) } else { otherNodes = append(otherNodes, &nodes.Items[i]) diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go index 49ceb84a55d0..02d3b75154ff 100644 --- a/test/suites/drift/suite_test.go +++ b/test/suites/drift/suite_test.go @@ -17,14 +17,18 @@ package drift_test import ( "fmt" "sort" - "strconv" - "strings" "testing" "time" + awssdk "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/eks" + "github.com/awslabs/operatorpkg/object" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -32,29 +36,20 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/karpenter/pkg/utils/resources" - - awssdk "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/eks" - "github.com/aws/aws-sdk-go/service/ssm" - - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coretest "sigs.k8s.io/karpenter/pkg/test" + "sigs.k8s.io/karpenter/pkg/utils/resources" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/test" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" "github.com/aws/karpenter-provider-aws/test/pkg/environment/common" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" ) var env *aws.Environment var amdAMI string -var nodeClass *v1beta1.EC2NodeClass -var nodePool *corev1beta1.NodePool +var nodeClass *v1.EC2NodeClass +var nodePool *karpv1.NodePool func TestDrift(t *testing.T) { RegisterFailHandler(Fail) @@ -80,7 +75,7 @@ var _ = Describe("Drift", func() { var selector labels.Selector var numPods int BeforeEach(func() { - amdAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", 1) + amdAMI = env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersion())) numPods = 1 // Add pods with a do-not-disrupt annotation so that we can check node metadata before we disrupt dep = coretest.Deployment(coretest.DeploymentOptions{ @@ -91,7 +86,7 @@ var _ = Describe("Drift", func() { "app": "my-app", }, Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", + karpv1.DoNotDisruptAnnotationKey: "true", }, }, TerminationGracePeriodSeconds: lo.ToPtr[int64](0), @@ -102,16 +97,16 @@ var _ = Describe("Drift", func() { Context("Budgets", func() { It("should respect budgets for empty drift", func() { nodePool = coretest.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceSize, + Operator: corev1.NodeSelectorOpIn, Values: []string{"2xlarge"}, }, }, ) // We're expecting to create 3 nodes, so we'll expect to see 2 nodes deleting at one time. - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{{ Nodes: "50%", }} var numPods int32 = 6 @@ -120,14 +115,14 @@ var _ = Describe("Drift", func() { PodOptions: coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", + karpv1.DoNotDisruptAnnotationKey: "true", }, Labels: map[string]string{"app": "large-app"}, }, // Each 2xlarge has 8 cpu, so each node should fit 2 pods. - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("3"), + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), }, }, }, @@ -179,16 +174,16 @@ var _ = Describe("Drift", func() { }) It("should respect budgets for non-empty delete drift", func() { nodePool = coretest.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceSize, + Operator: corev1.NodeSelectorOpIn, Values: []string{"2xlarge"}, }, }, ) // We're expecting to create 3 nodes, so we'll expect to see at most 2 nodes deleting at one time. - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{{ Nodes: "50%", }} var numPods int32 = 9 @@ -197,14 +192,14 @@ var _ = Describe("Drift", func() { PodOptions: coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", + karpv1.DoNotDisruptAnnotationKey: "true", }, Labels: map[string]string{"app": "large-app"}, }, // Each 2xlarge has 8 cpu, so each node should fit no more than 3 pods. - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2100m"), + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2100m"), }, }, }, @@ -245,7 +240,7 @@ var _ = Describe("Drift", func() { pods := env.EventuallyExpectHealthyPodCount(selector, 3) // Remove the do-not-disrupt annotation so that the nodes are now disruptable for _, pod := range pods { - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + delete(pod.Annotations, karpv1.DoNotDisruptAnnotationKey) env.ExpectUpdated(pod) } @@ -263,56 +258,34 @@ var _ = Describe("Drift", func() { }) It("should respect budgets for non-empty replace drift", func() { appLabels := map[string]string{"app": "large-app"} - - nodePool = coretest.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"xlarge"}, - }, - }, - // Add an Exists operator so that we can select on a fake partition later - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: "test-partition", - Operator: v1.NodeSelectorOpExists, - }, - }, - ) nodePool.Labels = appLabels // We're expecting to create 5 nodes, so we'll expect to see at most 3 nodes deleting at one time. - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{{ Nodes: "3", }} - // Make 5 pods all with different deployments and different test partitions, so that each pod can be put - // on a separate node. - selector = labels.SelectorFromSet(appLabels) + // Create a 5 pod deployment with hostname inter-pod anti-affinity to ensure each pod is placed on a unique node numPods = 5 - deployments := make([]*appsv1.Deployment, numPods) - for i := range lo.Range(numPods) { - deployments[i] = coretest.Deployment(coretest.DeploymentOptions{ - Replicas: 1, - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: appLabels, - }, - NodeSelector: map[string]string{"test-partition": fmt.Sprintf("%d", i)}, - // Each xlarge has 4 cpu, so each node should fit no more than 1 pod. - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("3"), - }, - }, + selector = labels.SelectorFromSet(appLabels) + deployment := coretest.Deployment(coretest.DeploymentOptions{ + Replicas: int32(numPods), + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: appLabels, }, - }) - } + PodAntiRequirements: []corev1.PodAffinityTerm{{ + TopologyKey: corev1.LabelHostname, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: appLabels, + }, + }}, + }, + }) - env.ExpectCreated(nodeClass, nodePool, deployments[0], deployments[1], deployments[2], deployments[3], deployments[4]) + env.ExpectCreated(nodeClass, nodePool, deployment) - originalNodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 5) - originalNodes := env.EventuallyExpectCreatedNodeCount("==", 5) + originalNodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", numPods) + originalNodes := env.EventuallyExpectCreatedNodeCount("==", numPods) // Check that all deployment pods are online env.EventuallyExpectHealthyPodCount(selector, numPods) @@ -342,14 +315,14 @@ var _ = Describe("Drift", func() { // Eventually expect all the nodes to be rolled and completely removed // Since this completes the disruption operation, this also ensures that we aren't leaking nodes into subsequent // tests since nodeclaims that are actively replacing but haven't brought-up nodes yet can register nodes later - env.EventuallyExpectNotFound(lo.Map(originalNodes, func(n *v1.Node, _ int) client.Object { return n })...) - env.EventuallyExpectNotFound(lo.Map(originalNodeClaims, func(n *corev1beta1.NodeClaim, _ int) client.Object { return n })...) + env.EventuallyExpectNotFound(lo.Map(originalNodes, func(n *corev1.Node, _ int) client.Object { return n })...) + env.EventuallyExpectNotFound(lo.Map(originalNodeClaims, func(n *karpv1.NodeClaim, _ int) client.Object { return n })...) env.ExpectNodeClaimCount("==", 5) env.ExpectNodeCount("==", 5) }) It("should not allow drift if the budget is fully blocking", func() { // We're going to define a budget that doesn't allow any drift to happen - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{{ Nodes: "0", }} @@ -374,7 +347,7 @@ var _ = Describe("Drift", func() { // the current time and extends 15 minutes past the current time // Times need to be in UTC since the karpenter containers were built in UTC time windowStart := time.Now().Add(-time.Minute * 15).UTC() - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{{ Nodes: "0", Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())), Duration: &metav1.Duration{Duration: time.Minute * 30}, @@ -397,16 +370,9 @@ var _ = Describe("Drift", func() { }) }) It("should disrupt nodes that have drifted due to AMIs", func() { - // Choose and old, static image. The 1.23 image is incompatible with EKS 1.29 so fallback to a newer image. - parameterName := lo.Ternary(lo.Must(strconv.Atoi(strings.Split(env.GetK8sVersion(0), ".")[1])) >= 29, - "/aws/service/eks/optimized-ami/1.27/amazon-linux-2023/x86_64/standard/amazon-eks-node-al2023-x86_64-standard-1.27-v20240307/image_id", - "/aws/service/eks/optimized-ami/1.23/amazon-linux-2023/arm64/standard/amazon-eks-node-al2023-arm64-standard-1.23-v20240307/image_id", - ) - parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{Name: awssdk.String(parameterName)}) - Expect(err).To(BeNil()) - oldCustomAMI := *parameter.Parameter.Value - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: oldCustomAMI}} + oldCustomAMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersionWithOffset(1))) + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: oldCustomAMI}} env.ExpectCreated(dep, nodeClass, nodePool) pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0] @@ -414,25 +380,20 @@ var _ = Describe("Drift", func() { nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] node := env.EventuallyExpectNodeCount("==", 1)[0] - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: amdAMI}} + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: amdAMI}} env.ExpectCreatedOrUpdated(nodeClass) env.EventuallyExpectDrifted(nodeClaim) - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + delete(pod.Annotations, karpv1.DoNotDisruptAnnotationKey) env.ExpectUpdated(pod) env.EventuallyExpectNotFound(pod, nodeClaim, node) env.EventuallyExpectHealthyPodCount(selector, numPods) }) It("should return drifted if the AMI no longer matches the existing NodeClaims instance type", func() { - version := env.GetK8sVersion(1) - armParameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ - Name: awssdk.String(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version)), - }) - Expect(err).To(BeNil()) - armAMI := *armParameter.Parameter.Value - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: armAMI}} + armAMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", env.K8sVersion())) + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: armAMI}} env.ExpectCreated(dep, nodeClass, nodePool) pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0] @@ -440,43 +401,34 @@ var _ = Describe("Drift", func() { nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] node := env.EventuallyExpectNodeCount("==", 1)[0] - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: amdAMI}} + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: amdAMI}} env.ExpectCreatedOrUpdated(nodeClass) env.EventuallyExpectDrifted(nodeClaim) - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + delete(pod.Annotations, karpv1.DoNotDisruptAnnotationKey) env.ExpectUpdated(pod) env.EventuallyExpectNotFound(pod, nodeClaim, node) env.EventuallyExpectHealthyPodCount(selector, numPods) }) It("should not disrupt nodes that have drifted without the featureGate enabled", func() { - version := env.GetK8sVersion(1) - env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=false"}) - // Choose an old static image (AL2023 AMIs don't exist for 1.22) - parameterName := lo.Ternary(lo.Must(strconv.Atoi(strings.Split(env.GetK8sVersion(0), ".")[1])) == 23, - "/aws/service/eks/optimized-ami/1.23/amazon-linux-2023/arm64/standard/amazon-eks-node-al2023-arm64-standard-1.23-v20240307/image_id", - fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version), - ) - parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ - Name: awssdk.String(parameterName), - }) - Expect(err).To(BeNil()) - oldCustomAMI := *parameter.Parameter.Value - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: oldCustomAMI}} + env.ExpectSettingsOverridden(corev1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=false"}) + + oldCustomAMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersionWithOffset(1))) + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: oldCustomAMI}} env.ExpectCreated(dep, nodeClass, nodePool) env.EventuallyExpectHealthyPodCount(selector, numPods) env.ExpectCreatedNodeCount("==", 1) node := env.Monitor.CreatedNodes()[0] - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: amdAMI}} + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: amdAMI}} env.ExpectUpdated(nodeClass) // We should consistently get the same node existing for a minute Consistently(func(g Gomega) { - g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), &v1.Node{})).To(Succeed()) + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), &corev1.Node{})).To(Succeed()) }).WithTimeout(time.Minute).Should(Succeed()) }) It("should disrupt nodes that have drifted due to securitygroup", func() { @@ -529,9 +481,9 @@ var _ = Describe("Drift", func() { } return "", false }) - sgTerms := []v1beta1.SecurityGroupSelectorTerm{{ID: awssdk.StringValue(testSecurityGroup.GroupId)}} + sgTerms := []v1.SecurityGroupSelectorTerm{{ID: awssdk.StringValue(testSecurityGroup.GroupId)}} for _, id := range awsIDs { - sgTerms = append(sgTerms, v1beta1.SecurityGroupSelectorTerm{ID: id}) + sgTerms = append(sgTerms, v1.SecurityGroupSelectorTerm{ID: id}) } nodeClass.Spec.SecurityGroupSelectorTerms = sgTerms @@ -540,7 +492,7 @@ var _ = Describe("Drift", func() { nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] node := env.ExpectCreatedNodeCount("==", 1)[0] - sgTerms = lo.Reject(sgTerms, func(t v1beta1.SecurityGroupSelectorTerm, _ int) bool { + sgTerms = lo.Reject(sgTerms, func(t v1.SecurityGroupSelectorTerm, _ int) bool { return t.ID == awssdk.StringValue(testSecurityGroup.GroupId) }) nodeClass.Spec.SecurityGroupSelectorTerms = sgTerms @@ -548,45 +500,51 @@ var _ = Describe("Drift", func() { env.EventuallyExpectDrifted(nodeClaim) - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + delete(pod.Annotations, karpv1.DoNotDisruptAnnotationKey) env.ExpectUpdated(pod) env.EventuallyExpectNotFound(pod, nodeClaim, node) env.EventuallyExpectHealthyPodCount(selector, numPods) }) It("should disrupt nodes that have drifted due to subnets", func() { - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": env.ClusterName}) + subnets := env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(subnets)).To(BeNumerically(">", 1)) - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{{ID: subnets[0].ID}} + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{{ID: subnets[0].ID}} env.ExpectCreated(dep, nodeClass, nodePool) pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0] nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] node := env.ExpectCreatedNodeCount("==", 1)[0] - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{{ID: subnets[1].ID}} + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{{ID: subnets[1].ID}} env.ExpectCreatedOrUpdated(nodeClass) env.EventuallyExpectDrifted(nodeClaim) - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + delete(pod.Annotations, karpv1.DoNotDisruptAnnotationKey) env.ExpectUpdated(pod) env.EventuallyExpectNotFound(pod, node) env.EventuallyExpectHealthyPodCount(selector, numPods) }) - DescribeTable("NodePool Drift", func(nodeClaimTemplate corev1beta1.NodeClaimTemplate) { + DescribeTable("NodePool Drift", func(nodeClaimTemplate karpv1.NodeClaimTemplate) { updatedNodePool := coretest.NodePool( - corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{Name: nodeClass.Name}, + karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + // keep the same instance type requirements to prevent considering instance types that require swap + Requirements: nodePool.Spec.Template.Spec.Requirements, }, }, }, }, - corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ + karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ Template: nodeClaimTemplate, }, }, @@ -602,7 +560,7 @@ var _ = Describe("Drift", func() { env.EventuallyExpectDrifted(nodeClaim) - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + delete(pod.Annotations, karpv1.DoNotDisruptAnnotationKey) env.ExpectUpdated(pod) // Nodes will need to have the start-up taint removed before the node can be considered as initialized @@ -616,50 +574,52 @@ var _ = Describe("Drift", func() { // Remove the startup taints from the new nodes to initialize them Eventually(func(g Gomega) { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeTwo), nodeTwo)).To(Succeed()) + g.Expect(len(nodeTwo.Spec.Taints)).To(BeNumerically("==", 1)) + _, found := lo.Find(nodeTwo.Spec.Taints, func(t corev1.Taint) bool { + return t.MatchTaint(&corev1.Taint{Key: "example.com/another-taint-2", Effect: corev1.TaintEffectPreferNoSchedule}) + }) + g.Expect(found).To(BeTrue()) stored := nodeTwo.DeepCopy() - nodeTwo.Spec.Taints = lo.Reject(nodeTwo.Spec.Taints, func(t v1.Taint, _ int) bool { return t.Key == "example.com/another-taint-2" }) + nodeTwo.Spec.Taints = lo.Reject(nodeTwo.Spec.Taints, func(t corev1.Taint, _ int) bool { return t.Key == "example.com/another-taint-2" }) g.Expect(env.Client.Patch(env.Context, nodeTwo, client.StrategicMergeFrom(stored))).To(Succeed()) }).Should(Succeed()) } env.EventuallyExpectNotFound(pod, node) env.EventuallyExpectHealthyPodCount(selector, numPods) }, - Entry("Annotations", corev1beta1.NodeClaimTemplate{ - ObjectMeta: corev1beta1.ObjectMeta{ + Entry("Annotations", karpv1.NodeClaimTemplate{ + ObjectMeta: karpv1.ObjectMeta{ Annotations: map[string]string{"keyAnnotationTest": "valueAnnotationTest"}, }, }), - Entry("Labels", corev1beta1.NodeClaimTemplate{ - ObjectMeta: corev1beta1.ObjectMeta{ + Entry("Labels", karpv1.NodeClaimTemplate{ + ObjectMeta: karpv1.ObjectMeta{ Labels: map[string]string{"keyLabelTest": "valueLabelTest"}, }, }), - Entry("Taints", corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - Taints: []v1.Taint{{Key: "example.com/another-taint-2", Effect: v1.TaintEffectPreferNoSchedule}}, + Entry("Taints", karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + Taints: []corev1.Taint{{Key: "example.com/another-taint-2", Effect: corev1.TaintEffectPreferNoSchedule}}, }, }), - Entry("KubeletConfiguration", corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - Kubelet: &corev1beta1.KubeletConfiguration{ - EvictionSoft: map[string]string{"memory.available": "5%"}, - EvictionSoftGracePeriod: map[string]metav1.Duration{"memory.available": {Duration: time.Minute}}, - }, - }, - }), - Entry("Start-up Taints", corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - StartupTaints: []v1.Taint{{Key: "example.com/another-taint-2", Effect: v1.TaintEffectPreferNoSchedule}}, + Entry("Start-up Taints", karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + StartupTaints: []corev1.Taint{{Key: "example.com/another-taint-2", Effect: corev1.TaintEffectPreferNoSchedule}}, }, }), - Entry("NodeRequirements", corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot}}}}, + Entry("NodeRequirements", karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + // since this will overwrite the default requirements, add instance category and family selectors back into requirements + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: karpv1.CapacityTypeLabelKey, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.CapacityTypeSpot}}}, + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: v1.LabelInstanceCategory, Operator: corev1.NodeSelectorOpIn, Values: []string{"c", "m", "r"}}}, + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: v1.LabelInstanceFamily, Operator: corev1.NodeSelectorOpNotIn, Values: []string{"a1"}}}, + }, }, }), ) - DescribeTable("EC2NodeClass", func(nodeClassSpec v1beta1.EC2NodeClassSpec) { - updatedNodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{Spec: *nodeClass.Spec.DeepCopy()}, v1beta1.EC2NodeClass{Spec: nodeClassSpec}) + DescribeTable("EC2NodeClass", func(nodeClassSpec v1.EC2NodeClassSpec) { + updatedNodeClass := test.EC2NodeClass(v1.EC2NodeClass{Spec: *nodeClass.Spec.DeepCopy()}, v1.EC2NodeClass{Spec: nodeClassSpec}) updatedNodeClass.ObjectMeta = nodeClass.ObjectMeta env.ExpectCreated(dep, nodeClass, nodePool) @@ -671,25 +631,33 @@ var _ = Describe("Drift", func() { env.EventuallyExpectDrifted(nodeClaim) - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + delete(pod.Annotations, karpv1.DoNotDisruptAnnotationKey) env.ExpectUpdated(pod) env.EventuallyExpectNotFound(pod, node) env.EventuallyExpectHealthyPodCount(selector, numPods) }, - Entry("UserData", v1beta1.EC2NodeClassSpec{UserData: awssdk.String("#!/bin/bash\necho \"Hello, AL2023\"")}), - Entry("Tags", v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}), - Entry("MetadataOptions", v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPTokens: awssdk.String("required"), HTTPPutResponseHopLimit: awssdk.Int64(10)}}), - Entry("BlockDeviceMappings", v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + Entry("UserData", v1.EC2NodeClassSpec{UserData: awssdk.String("#!/bin/bash\necho \"Hello, AL2023\"")}), + Entry("Tags", v1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}), + Entry("MetadataOptions", v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPTokens: awssdk.String("required"), HTTPPutResponseHopLimit: awssdk.Int64(10)}}), + Entry("BlockDeviceMappings", v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{ { DeviceName: awssdk.String("/dev/xvda"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ VolumeSize: resources.Quantity("20Gi"), VolumeType: awssdk.String("gp3"), Encrypted: awssdk.Bool(true), }, }}}), - Entry("DetailedMonitoring", v1beta1.EC2NodeClassSpec{DetailedMonitoring: awssdk.Bool(true)}), - Entry("AMIFamily", v1beta1.EC2NodeClassSpec{AMIFamily: awssdk.String(v1beta1.AMIFamilyBottlerocket)}), + Entry("DetailedMonitoring", v1.EC2NodeClassSpec{DetailedMonitoring: awssdk.Bool(true)}), + Entry("AMIFamily", v1.EC2NodeClassSpec{ + AMISelectorTerms: []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}}, + }), + Entry("KubeletConfiguration", v1.EC2NodeClassSpec{ + Kubelet: &v1.KubeletConfiguration{ + EvictionSoft: map[string]string{"memory.available": "5%"}, + EvictionSoftGracePeriod: map[string]metav1.Duration{"memory.available": {Duration: time.Minute}}, + }, + }), ) It("should drift the EC2NodeClass on InstanceProfile", func() { // Create a separate test case for this one since we can't use the default NodeClass that's created due to it having @@ -717,16 +685,16 @@ var _ = Describe("Drift", func() { env.EventuallyExpectDrifted(nodeClaim) - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + delete(pod.Annotations, karpv1.DoNotDisruptAnnotationKey) env.ExpectUpdated(pod) env.EventuallyExpectNotFound(pod, node) env.EventuallyExpectHealthyPodCount(selector, numPods) }) It("should drift the EC2NodeClass on BlockDeviceMappings volume size update", func() { - nodeClass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ + nodeClass.Spec.BlockDeviceMappings = []*v1.BlockDeviceMapping{ { DeviceName: awssdk.String("/dev/xvda"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ VolumeSize: resources.Quantity("20Gi"), VolumeType: awssdk.String("gp3"), Encrypted: awssdk.Bool(true), @@ -744,11 +712,11 @@ var _ = Describe("Drift", func() { By("validating the drifted status condition has propagated") Eventually(func(g Gomega) { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) - g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Drifted)).ToNot(BeNil()) - g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Drifted).IsTrue()).To(BeTrue()) + g.Expect(nodeClaim.StatusConditions().Get(karpv1.ConditionTypeDrifted)).ToNot(BeNil()) + g.Expect(nodeClaim.StatusConditions().Get(karpv1.ConditionTypeDrifted).IsTrue()).To(BeTrue()) }).Should(Succeed()) - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + delete(pod.Annotations, karpv1.DoNotDisruptAnnotationKey) env.ExpectUpdated(pod) env.EventuallyExpectNotFound(pod, node) env.EventuallyExpectHealthyPodCount(selector, numPods) @@ -757,23 +725,23 @@ var _ = Describe("Drift", func() { env.ExpectCreated(dep, nodeClass, nodePool) env.EventuallyExpectHealthyPodCount(selector, numPods) nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] - nodePool = env.ExpectExists(nodePool).(*corev1beta1.NodePool) + nodePool = env.ExpectExists(nodePool).(*karpv1.NodePool) expectedHash := nodePool.Hash() - By(fmt.Sprintf("expect nodepool %s and nodeclaim %s to contain %s and %s annotations", nodePool.Name, nodeClaim.Name, corev1beta1.NodePoolHashAnnotationKey, corev1beta1.NodePoolHashVersionAnnotationKey)) + By(fmt.Sprintf("expect nodepool %s and nodeclaim %s to contain %s and %s annotations", nodePool.Name, nodeClaim.Name, karpv1.NodePoolHashAnnotationKey, karpv1.NodePoolHashVersionAnnotationKey)) Eventually(func(g Gomega) { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodePool), nodePool)).To(Succeed()) g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) - g.Expect(nodePool.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashAnnotationKey, expectedHash)) - g.Expect(nodePool.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashVersionAnnotationKey, corev1beta1.NodePoolHashVersion)) - g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashAnnotationKey, expectedHash)) - g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashVersionAnnotationKey, corev1beta1.NodePoolHashVersion)) + g.Expect(nodePool.Annotations).To(HaveKeyWithValue(karpv1.NodePoolHashAnnotationKey, expectedHash)) + g.Expect(nodePool.Annotations).To(HaveKeyWithValue(karpv1.NodePoolHashVersionAnnotationKey, karpv1.NodePoolHashVersion)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(karpv1.NodePoolHashAnnotationKey, expectedHash)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(karpv1.NodePoolHashVersionAnnotationKey, karpv1.NodePoolHashVersion)) }).WithTimeout(30 * time.Second).Should(Succeed()) nodePool.Annotations = lo.Assign(nodePool.Annotations, map[string]string{ - corev1beta1.NodePoolHashAnnotationKey: "test-hash-1", - corev1beta1.NodePoolHashVersionAnnotationKey: "test-hash-version-1", + karpv1.NodePoolHashAnnotationKey: "test-hash-1", + karpv1.NodePoolHashVersionAnnotationKey: "test-hash-version-1", }) // Updating `nodePool.Spec.Template.Annotations` would normally trigger drift on all nodeclaims owned by the // nodepool. However, the nodepool-hash-version does not match the controller hash version, so we will see that @@ -782,8 +750,8 @@ var _ = Describe("Drift", func() { "test-key": "test-value", }) nodeClaim.Annotations = lo.Assign(nodePool.Annotations, map[string]string{ - corev1beta1.NodePoolHashAnnotationKey: "test-hash-2", - corev1beta1.NodePoolHashVersionAnnotationKey: "test-hash-version-2", + karpv1.NodePoolHashAnnotationKey: "test-hash-2", + karpv1.NodePoolHashVersionAnnotationKey: "test-hash-version-2", }) // The nodeclaim will need to be updated first, as the hash controller will only be triggered on changes to the nodepool @@ -795,33 +763,33 @@ var _ = Describe("Drift", func() { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodePool), nodePool)).To(Succeed()) g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) - g.Expect(nodePool.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashAnnotationKey, expectedHash)) - g.Expect(nodePool.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashVersionAnnotationKey, corev1beta1.NodePoolHashVersion)) - g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashAnnotationKey, expectedHash)) - g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashVersionAnnotationKey, corev1beta1.NodePoolHashVersion)) + g.Expect(nodePool.Annotations).To(HaveKeyWithValue(karpv1.NodePoolHashAnnotationKey, expectedHash)) + g.Expect(nodePool.Annotations).To(HaveKeyWithValue(karpv1.NodePoolHashVersionAnnotationKey, karpv1.NodePoolHashVersion)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(karpv1.NodePoolHashAnnotationKey, expectedHash)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(karpv1.NodePoolHashVersionAnnotationKey, karpv1.NodePoolHashVersion)) }) }) It("should update the ec2nodeclass-hash annotation on the ec2nodeclass and nodeclaim when the ec2nodeclass's ec2nodeclass-hash-version annotation does not match the controller hash version", func() { env.ExpectCreated(dep, nodeClass, nodePool) env.EventuallyExpectHealthyPodCount(selector, numPods) nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] - nodeClass = env.ExpectExists(nodeClass).(*v1beta1.EC2NodeClass) + nodeClass = env.ExpectExists(nodeClass).(*v1.EC2NodeClass) expectedHash := nodeClass.Hash() - By(fmt.Sprintf("expect nodeclass %s and nodeclaim %s to contain %s and %s annotations", nodeClass.Name, nodeClaim.Name, v1beta1.AnnotationEC2NodeClassHash, v1beta1.AnnotationEC2NodeClassHashVersion)) + By(fmt.Sprintf("expect nodeclass %s and nodeclaim %s to contain %s and %s annotations", nodeClass.Name, nodeClaim.Name, v1.AnnotationEC2NodeClassHash, v1.AnnotationEC2NodeClassHashVersion)) Eventually(func(g Gomega) { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClass), nodeClass)).To(Succeed()) g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) - g.Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) - g.Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) - g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) - g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + g.Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHash, expectedHash)) + g.Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHashVersion, v1.EC2NodeClassHashVersion)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHash, expectedHash)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHashVersion, v1.EC2NodeClassHashVersion)) }).WithTimeout(30 * time.Second).Should(Succeed()) nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "test-hash-1", - v1beta1.AnnotationEC2NodeClassHashVersion: "test-hash-version-1", + v1.AnnotationEC2NodeClassHash: "test-hash-1", + v1.AnnotationEC2NodeClassHashVersion: "test-hash-version-1", }) // Updating `nodeClass.Spec.Tags` would normally trigger drift on all nodeclaims using the // nodeclass. However, the ec2nodeclass-hash-version does not match the controller hash version, so we will see that @@ -830,8 +798,8 @@ var _ = Describe("Drift", func() { "test-key": "test-value", }) nodeClaim.Annotations = lo.Assign(nodePool.Annotations, map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: "test-hash-2", - v1beta1.AnnotationEC2NodeClassHashVersion: "test-hash-version-2", + v1.AnnotationEC2NodeClassHash: "test-hash-2", + v1.AnnotationEC2NodeClassHashVersion: "test-hash-version-2", }) // The nodeclaim will need to be updated first, as the hash controller will only be triggered on changes to the nodeclass @@ -843,23 +811,23 @@ var _ = Describe("Drift", func() { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClass), nodeClass)).To(Succeed()) g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) - g.Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) - g.Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) - g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) - g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + g.Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHash, expectedHash)) + g.Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHashVersion, v1.EC2NodeClassHashVersion)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHash, expectedHash)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationEC2NodeClassHashVersion, v1.EC2NodeClassHashVersion)) }).WithTimeout(30 * time.Second).Should(Succeed()) env.ConsistentlyExpectNodeClaimsNotDrifted(time.Minute, nodeClaim) }) Context("Failure", func() { - It("should not continue to drift if a node never registers", func() { + It("should not disrupt a drifted node if the replacement node never registers", func() { // launch a new nodeClaim var numPods int32 = 2 dep := coretest.Deployment(coretest.DeploymentOptions{ Replicas: 2, PodOptions: coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "inflate"}}, - PodAntiRequirements: []v1.PodAffinityTerm{{ - TopologyKey: v1.LabelHostname, + PodAntiRequirements: []corev1.PodAffinityTerm{{ + TopologyKey: corev1.LabelHostname, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{"app": "inflate"}, }}, @@ -872,48 +840,39 @@ var _ = Describe("Drift", func() { env.EventuallyExpectCreatedNodeCount("==", int(numPods)) // Drift the nodeClaim with bad configuration that will not register a NodeClaim - parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ - Name: awssdk.String("/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs"), - }) - Expect(err).ToNot(HaveOccurred()) - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: *parameter.Parameter.Value}} + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: env.GetAMIBySSMPath("/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs")}} env.ExpectCreatedOrUpdated(nodeClass) env.EventuallyExpectDrifted(startingNodeClaimState...) - // Expect nodes To get tainted + // Expect only a single node to be tainted due to default disruption budgets taintedNodes := env.EventuallyExpectTaintedNodeCount("==", 1) // Drift should fail and the original node should be untainted // TODO: reduce timeouts when disruption waits are factored out env.EventuallyExpectNodesUntaintedWithTimeout(11*time.Minute, taintedNodes...) - // We give another 6 minutes here to handle the deletion at the 15m registration timeout - Eventually(func(g Gomega) { - nodeClaims := &corev1beta1.NodeClaimList{} - g.Expect(env.Client.List(env, nodeClaims, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) - g.Expect(nodeClaims.Items).To(HaveLen(int(numPods))) - }).WithTimeout(6 * time.Minute).Should(Succeed()) - - // Expect all the NodeClaims that existed on the initial provisioning loop are not removed + // Expect all the NodeClaims that existed on the initial provisioning loop are not removed. + // Assert this over several minutes to ensure a subsequent disruption controller pass doesn't + // successfully schedule the evicted pods to the in-flight nodeclaim and disrupt the original node Consistently(func(g Gomega) { - nodeClaims := &corev1beta1.NodeClaimList{} + nodeClaims := &karpv1.NodeClaimList{} g.Expect(env.Client.List(env, nodeClaims, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) - - startingNodeClaimUIDs := lo.Map(startingNodeClaimState, func(nc *corev1beta1.NodeClaim, _ int) types.UID { return nc.UID }) - nodeClaimUIDs := lo.Map(nodeClaims.Items, func(nc corev1beta1.NodeClaim, _ int) types.UID { return nc.UID }) - g.Expect(sets.New(nodeClaimUIDs...).IsSuperset(sets.New(startingNodeClaimUIDs...))).To(BeTrue()) + startingNodeClaimUIDs := sets.New(lo.Map(startingNodeClaimState, func(nc *karpv1.NodeClaim, _ int) types.UID { return nc.UID })...) + nodeClaimUIDs := sets.New(lo.Map(nodeClaims.Items, func(nc karpv1.NodeClaim, _ int) types.UID { return nc.UID })...) + g.Expect(nodeClaimUIDs.IsSuperset(startingNodeClaimUIDs)).To(BeTrue()) }, "2m").Should(Succeed()) }) - It("should not continue to drift if a node registers but never becomes initialized", func() { + It("should not disrupt a drifted node if the replacement node registers but never initialized", func() { // launch a new nodeClaim var numPods int32 = 2 dep := coretest.Deployment(coretest.DeploymentOptions{ Replicas: 2, PodOptions: coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "inflate"}}, - PodAntiRequirements: []v1.PodAffinityTerm{{ - TopologyKey: v1.LabelHostname, + PodAntiRequirements: []corev1.PodAffinityTerm{{ + TopologyKey: corev1.LabelHostname, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{"app": "inflate"}, }}, @@ -926,12 +885,12 @@ var _ = Describe("Drift", func() { env.EventuallyExpectCreatedNodeCount("==", int(numPods)) // Drift the nodeClaim with bad configuration that never initializes - nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com/taint", Effect: v1.TaintEffectPreferNoSchedule}} + nodePool.Spec.Template.Spec.StartupTaints = []corev1.Taint{{Key: "example.com/taint", Effect: corev1.TaintEffectPreferNoSchedule}} env.ExpectCreatedOrUpdated(nodePool) env.EventuallyExpectDrifted(startingNodeClaimState...) - // Expect nodes to be tainted + // Expect only a single node to get tainted due to default disruption budgets taintedNodes := env.EventuallyExpectTaintedNodeCount("==", 1) // Drift should fail and original node should be untainted @@ -939,22 +898,23 @@ var _ = Describe("Drift", func() { env.EventuallyExpectNodesUntaintedWithTimeout(11*time.Minute, taintedNodes...) // Expect that the new nodeClaim/node is kept around after the un-cordon - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} Expect(env.Client.List(env, nodeList, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) Expect(nodeList.Items).To(HaveLen(int(numPods) + 1)) - nodeClaimList := &corev1beta1.NodeClaimList{} + nodeClaimList := &karpv1.NodeClaimList{} Expect(env.Client.List(env, nodeClaimList, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) Expect(nodeClaimList.Items).To(HaveLen(int(numPods) + 1)) // Expect all the NodeClaims that existed on the initial provisioning loop are not removed + // Assert this over several minutes to ensure a subsequent disruption controller pass doesn't + // successfully schedule the evicted pods to the in-flight nodeclaim and disrupt the original node Consistently(func(g Gomega) { - nodeClaims := &corev1beta1.NodeClaimList{} + nodeClaims := &karpv1.NodeClaimList{} g.Expect(env.Client.List(env, nodeClaims, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) - - startingNodeClaimUIDs := lo.Map(startingNodeClaimState, func(m *corev1beta1.NodeClaim, _ int) types.UID { return m.UID }) - nodeClaimUIDs := lo.Map(nodeClaims.Items, func(m corev1beta1.NodeClaim, _ int) types.UID { return m.UID }) - g.Expect(sets.New(nodeClaimUIDs...).IsSuperset(sets.New(startingNodeClaimUIDs...))).To(BeTrue()) + startingNodeClaimUIDs := sets.New(lo.Map(startingNodeClaimState, func(m *karpv1.NodeClaim, _ int) types.UID { return m.UID })...) + nodeClaimUIDs := sets.New(lo.Map(nodeClaims.Items, func(m karpv1.NodeClaim, _ int) types.UID { return m.UID })...) + g.Expect(nodeClaimUIDs.IsSuperset(startingNodeClaimUIDs)).To(BeTrue()) }, "2m").Should(Succeed()) }) It("should not drift any nodes if their PodDisruptionBudgets are unhealthy", func() { @@ -965,15 +925,15 @@ var _ = Describe("Drift", func() { Replicas: 2, PodOptions: coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "inflate"}}, - PodAntiRequirements: []v1.PodAffinityTerm{{ - TopologyKey: v1.LabelHostname, + PodAntiRequirements: []corev1.PodAffinityTerm{{ + TopologyKey: corev1.LabelHostname, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{"app": "inflate"}, }}, }, - ReadinessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - HTTPGet: &v1.HTTPGetAction{ + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ Port: intstr.FromInt32(80), }, }, diff --git a/test/suites/expiration/suite_test.go b/test/suites/expiration/suite_test.go index 168381716de6..7f8ccc09dffa 100644 --- a/test/suites/expiration/suite_test.go +++ b/test/suites/expiration/suite_test.go @@ -15,28 +15,21 @@ limitations under the License. package expiration_test import ( - "fmt" "testing" - "time" "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/aws/aws-sdk-go/service/ssm" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" - "github.com/aws/karpenter-provider-aws/test/pkg/environment/common" coretest "sigs.k8s.io/karpenter/pkg/test" @@ -45,8 +38,8 @@ import ( ) var env *aws.Environment -var nodeClass *v1beta1.EC2NodeClass -var nodePool *corev1beta1.NodePool +var nodeClass *v1.EC2NodeClass +var nodePool *karpv1.NodePool func TestExpiration(t *testing.T) { RegisterFailHandler(Fail) @@ -63,7 +56,6 @@ var _ = BeforeEach(func() { env.BeforeEach() nodeClass = env.DefaultEC2NodeClass() nodePool = env.DefaultNodePool(nodeClass) - nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Second * 30)} }) var _ = AfterEach(func() { env.Cleanup() }) @@ -75,7 +67,6 @@ var _ = Describe("Expiration", func() { var numPods int BeforeEach(func() { numPods = 1 - // Add pods with a do-not-disrupt annotation so that we can check node metadata before we disrupt dep = coretest.Deployment(coretest.DeploymentOptions{ Replicas: int32(numPods), PodOptions: coretest.PodOptions{ @@ -83,393 +74,14 @@ var _ = Describe("Expiration", func() { Labels: map[string]string{ "app": "my-app", }, - Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", - }, }, TerminationGracePeriodSeconds: lo.ToPtr[int64](0), }, }) selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) }) - Context("Budgets", func() { - // Two nodes, both expired or both drifted, the more drifted one with a pre-stop pod that sleeps for 300 seconds, - // and we consistently ensure that the second node is not tainted == disrupted. - It("should not continue to disrupt nodes that have been the target of pod nomination", func() { - coretest.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"2xlarge"}, - }, - }, - ) - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ - Nodes: "100%", - }} - nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} - - // Create a deployment with one pod to create one node. - dep = coretest.Deployment(coretest.DeploymentOptions{ - Replicas: 1, - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", - }, - Labels: map[string]string{"app": "large-app"}, - }, - // Each 2xlarge has 8 cpu, so each node should fit 2 pods. - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("3"), - }, - }, - Command: []string{"sh", "-c", "sleep 3600"}, - Image: "alpine:latest", - PreStopSleep: lo.ToPtr(int64(300)), - TerminationGracePeriodSeconds: lo.ToPtr(int64(500)), - }, - }) - selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) - env.ExpectCreated(nodeClass, nodePool, dep) - - env.EventuallyExpectCreatedNodeClaimCount("==", 1) - env.EventuallyExpectCreatedNodeCount("==", 1) - env.EventuallyExpectHealthyPodCount(selector, 1) - - // Set the node to unschedulable so that we can create another node with one pod. - node := env.EventuallyExpectNodeCount("==", 1)[0] - node.Spec.Unschedulable = true - env.ExpectUpdated(node) - - dep.Spec.Replicas = lo.ToPtr(int32(2)) - env.ExpectUpdated(dep) - - ncs := env.EventuallyExpectCreatedNodeClaimCount("==", 2) - env.EventuallyExpectCreatedNodeCount("==", 2) - pods := env.EventuallyExpectHealthyPodCount(selector, 2) - env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration - - node = env.ExpectExists(node).(*v1.Node) - node.Spec.Unschedulable = false - env.ExpectUpdated(node) - - By("enabling expiration") - nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Second * 30)} - env.ExpectUpdated(nodePool) - - // Expect that both of the nodes are expired, but not being disrupted - env.EventuallyExpectExpired(ncs...) - env.ConsistentlyExpectNoDisruptions(2, 30*time.Second) - - By("removing the do not disrupt annotations") - // Remove the do not disrupt annotation from the two pods - for _, p := range pods { - p := env.ExpectExists(p).(*v1.Pod) - delete(p.Annotations, corev1beta1.DoNotDisruptAnnotationKey) - env.ExpectUpdated(p) - } - env.EventuallyExpectTaintedNodeCount("==", 1) - - By("expecting only one disruption for 60s") - // Expect only one node being disrupted as the other node should continue to be nominated. - // As the pod has a 300s pre-stop sleep. - env.ConsistentlyExpectDisruptionsWithNodeCount(1, 2, time.Minute) - }) - It("should respect budgets for empty expiration", func() { - coretest.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"2xlarge"}, - }, - }, - ) - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ - Nodes: "50%", - }} - nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} - - numPods = 6 - dep = coretest.Deployment(coretest.DeploymentOptions{ - Replicas: int32(numPods), - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", - }, - Labels: map[string]string{"app": "large-app"}, - }, - // Each 2xlarge has 8 cpu, so each node should fit 2 pods. - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("3"), - }, - }, - }, - }) - selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) - env.ExpectCreated(nodeClass, nodePool, dep) - - nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 3) - nodes := env.EventuallyExpectCreatedNodeCount("==", 3) - env.EventuallyExpectHealthyPodCount(selector, numPods) - env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration - - By("adding finalizers to the nodes to prevent termination") - // Add a finalizer to each node so that we can stop termination disruptions - for _, node := range nodes { - Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) - node.Finalizers = append(node.Finalizers, common.TestingFinalizer) - env.ExpectUpdated(node) - } - - By("making the nodes empty") - // Delete the deployment to make all nodes empty. - env.ExpectDeleted(dep) - - By("enabling expiration") - nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Second * 30)} - env.ExpectUpdated(nodePool) - - env.EventuallyExpectExpired(nodeClaims...) - - // Expect that two nodes are tainted. - env.EventuallyExpectTaintedNodeCount("==", 2) - nodes = env.ConsistentlyExpectDisruptionsWithNodeCount(2, 3, 5*time.Second) - - // Remove finalizers - for _, node := range nodes { - Expect(env.ExpectTestingFinalizerRemoved(node)).To(Succeed()) - } - - // After the deletion timestamp is set and all pods are drained - // the node should be gone - env.EventuallyExpectNotFound(nodes[0], nodes[1]) - - // Expect that only one node is tainted, even considering the new node that was just created. - nodes = env.EventuallyExpectTaintedNodeCount("==", 1) - - // Expect the finalizers to be removed and deleted. - Expect(env.ExpectTestingFinalizerRemoved(nodes[0])).To(Succeed()) - env.EventuallyExpectNotFound(nodes[0]) - }) - It("should respect budgets for non-empty delete expiration", func() { - nodePool = coretest.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"2xlarge"}, - }, - }, - ) - // We're expecting to create 3 nodes, so we'll expect to see at most 2 nodes deleting at one time. - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ - Nodes: "50%", - }} - // disable expiration so that we can enable it later when we want. - nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} - numPods = 9 - dep = coretest.Deployment(coretest.DeploymentOptions{ - Replicas: int32(numPods), - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", - }, - Labels: map[string]string{"app": "large-app"}, - }, - // Each 2xlarge has 8 cpu, so each node should fit no more than 3 pods. - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2100m"), - }, - }, - }, - }) - selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) - env.ExpectCreated(nodeClass, nodePool, dep) - - nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 3) - nodes := env.EventuallyExpectCreatedNodeCount("==", 3) - env.EventuallyExpectHealthyPodCount(selector, numPods) - - By("scaling down the deployment") - // Update the deployment to a third of the replicas. - dep.Spec.Replicas = lo.ToPtr[int32](3) - env.ExpectUpdated(dep) - - // First expect there to be 3 pods, then try to spread the pods. - env.EventuallyExpectHealthyPodCount(selector, 3) - env.ForcePodsToSpread(nodes...) - env.EventuallyExpectHealthyPodCount(selector, 3) - - By("cordoning and adding finalizer to the nodes") - // Add a finalizer to each node so that we can stop termination disruptions - for _, node := range nodes { - Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) - node.Finalizers = append(node.Finalizers, common.TestingFinalizer) - env.ExpectUpdated(node) - } - - By("expiring the nodes") - // expire the nodeclaims - nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Second * 30)} - env.ExpectUpdated(nodePool) - - env.EventuallyExpectExpired(nodeClaims...) - - By("enabling disruption by removing the do not disrupt annotation") - pods := env.EventuallyExpectHealthyPodCount(selector, 3) - // Remove the do-not-disrupt annotation so that the nodes are now disruptable - for _, pod := range pods { - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) - env.ExpectUpdated(pod) - } - - // Ensure that we get two nodes tainted, and they have overlap during the expiration - env.EventuallyExpectTaintedNodeCount("==", 2) - nodes = env.ConsistentlyExpectDisruptionsWithNodeCount(2, 3, 5*time.Second) - - By("removing the finalizer from the nodes") - Expect(env.ExpectTestingFinalizerRemoved(nodes[0])).To(Succeed()) - Expect(env.ExpectTestingFinalizerRemoved(nodes[1])).To(Succeed()) - - // After the deletion timestamp is set and all pods are drained - // the node should be gone - env.EventuallyExpectNotFound(nodes[0], nodes[1]) - }) - It("should respect budgets for non-empty replace expiration", func() { - appLabels := map[string]string{"app": "large-app"} - - nodePool = coretest.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"xlarge"}, - }, - }, - // Add an Exists operator so that we can select on a fake partition later - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: "test-partition", - Operator: v1.NodeSelectorOpExists, - }, - }, - ) - nodePool.Labels = appLabels - // We're expecting to create 5 nodes, so we'll expect to see at most 3 nodes deleting at one time. - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ - Nodes: "3", - }} - - // Make 5 pods all with different deployments and different test partitions, so that each pod can be put - // on a separate node. - selector = labels.SelectorFromSet(appLabels) - numPods = 5 - deployments := make([]*appsv1.Deployment, numPods) - for i := range lo.Range(numPods) { - deployments[i] = coretest.Deployment(coretest.DeploymentOptions{ - Replicas: 1, - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: appLabels, - }, - NodeSelector: map[string]string{"test-partition": fmt.Sprintf("%d", i)}, - // Each xlarge has 4 cpu, so each node should fit no more than 1 pod. - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("3"), - }, - }, - }, - }) - } - - env.ExpectCreated(nodeClass, nodePool, deployments[0], deployments[1], deployments[2], deployments[3], deployments[4]) - - env.EventuallyExpectCreatedNodeClaimCount("==", 5) - nodes := env.EventuallyExpectCreatedNodeCount("==", 5) - - // Check that all daemonsets and deployment pods are online - env.EventuallyExpectHealthyPodCount(selector, numPods) - - By("cordoning and adding finalizer to the nodes") - // Add a finalizer to each node so that we can stop termination disruptions - for _, node := range nodes { - Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) - node.Finalizers = append(node.Finalizers, common.TestingFinalizer) - env.ExpectUpdated(node) - } - - By("enabling expiration") - nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{Duration: lo.ToPtr(30 * time.Second)} - env.ExpectUpdated(nodePool) - - // Ensure that we get two nodes tainted, and they have overlap during the expiration - env.EventuallyExpectTaintedNodeCount("==", 3) - env.EventuallyExpectNodeClaimCount("==", 8) - env.EventuallyExpectNodeCount("==", 8) - nodes = env.ConsistentlyExpectDisruptionsWithNodeCount(3, 8, 5*time.Second) - - // Set the expireAfter to "Never" to make sure new node isn't deleted - // This is CRITICAL since it prevents nodes that are immediately spun up from immediately being expired and - // racing at the end of the E2E test, leaking node resources into subsequent tests - nodePool.Spec.Disruption.ExpireAfter.Duration = nil - env.ExpectUpdated(nodePool) - - for _, node := range nodes { - Expect(env.ExpectTestingFinalizerRemoved(node)).To(Succeed()) - } - - env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2]) - env.ExpectNodeCount("==", 5) - }) - It("should not allow expiration if the budget is fully blocking", func() { - // We're going to define a budget that doesn't allow any expirations to happen - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ - Nodes: "0", - }} - - dep.Spec.Template.Annotations = nil - env.ExpectCreated(nodeClass, nodePool, dep) - - nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] - env.EventuallyExpectCreatedNodeCount("==", 1) - env.EventuallyExpectHealthyPodCount(selector, numPods) - - env.EventuallyExpectExpired(nodeClaim) - env.ConsistentlyExpectNoDisruptions(1, time.Minute) - }) - It("should not allow expiration if the budget is fully blocking during a scheduled time", func() { - // We're going to define a budget that doesn't allow any expirations to happen - // This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before - // the current time and extends 15 minutes past the current time - // Times need to be in UTC since the karpenter containers were built in UTC time - windowStart := time.Now().Add(-time.Minute * 15).UTC() - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ - Nodes: "0", - Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())), - Duration: &metav1.Duration{Duration: time.Minute * 30}, - }} - - dep.Spec.Template.Annotations = nil - env.ExpectCreated(nodeClass, nodePool, dep) - - nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] - env.EventuallyExpectCreatedNodeCount("==", 1) - env.EventuallyExpectHealthyPodCount(selector, numPods) - - env.EventuallyExpectExpired(nodeClaim) - env.ConsistentlyExpectNoDisruptions(1, time.Minute) - }) - }) It("should expire the node after the expiration is reached", func() { + nodePool.Spec.Template.Spec.ExpireAfter = karpv1.MustParseNillableDuration("2m") env.ExpectCreated(nodeClass, nodePool, dep) nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] @@ -477,27 +89,21 @@ var _ = Describe("Expiration", func() { env.EventuallyExpectHealthyPodCount(selector, numPods) env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration - env.EventuallyExpectExpired(nodeClaim) - - // Remove the do-not-disrupt annotation so that the Nodes are now deprovisionable - for _, pod := range env.ExpectPodsMatchingSelector(selector) { - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) - env.ExpectUpdated(pod) - } - // Eventually the node will be tainted, which means its actively being disrupted Eventually(func(g Gomega) { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).Should(Succeed()) - _, ok := lo.Find(node.Spec.Taints, func(t v1.Taint) bool { - return corev1beta1.IsDisruptingTaint(t) + _, ok := lo.Find(node.Spec.Taints, func(t corev1.Taint) bool { + return karpv1.IsDisruptingTaint(t) }) g.Expect(ok).To(BeTrue()) }).Should(Succeed()) - // Set the expireAfter to "Never" to make sure new node isn't deleted - // This is CRITICAL since it prevents nodes that are immediately spun up from immediately being expired and - // racing at the end of the E2E test, leaking node resources into subsequent tests - nodePool.Spec.Disruption.ExpireAfter.Duration = nil + env.EventuallyExpectCreatedNodeCount("==", 2) + // Set the limit to 0 to make sure we don't continue to create nodeClaims. + // This is CRITICAL since it prevents leaking node resources into subsequent tests + nodePool.Spec.Limits = karpv1.Limits{ + corev1.ResourceCPU: resource.MustParse("0"), + } env.ExpectUpdated(nodePool) // After the deletion timestamp is set and all pods are drained @@ -509,26 +115,19 @@ var _ = Describe("Expiration", func() { env.EventuallyExpectHealthyPodCount(selector, numPods) }) It("should replace expired node with a single node and schedule all pods", func() { + // Set expire after to 5 minutes since we have to respect PDB and move over pods one at a time from one node to another. + // The new nodes should not expire before all the pods are moved over. + nodePool.Spec.Template.Spec.ExpireAfter = karpv1.MustParseNillableDuration("5m") var numPods int32 = 5 // We should setup a PDB that will only allow a minimum of 1 pod to be pending at a time minAvailable := intstr.FromInt32(numPods - 1) pdb := coretest.PodDisruptionBudget(coretest.PDBOptions{ Labels: map[string]string{ - "app": "large-app", + "app": "my-app", }, MinAvailable: &minAvailable, }) - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: numPods, - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", - }, - Labels: map[string]string{"app": "large-app"}, - }, - }, - }) + dep.Spec.Replicas = &numPods selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) env.ExpectCreated(nodeClass, nodePool, pdb, dep) @@ -537,31 +136,21 @@ var _ = Describe("Expiration", func() { env.EventuallyExpectHealthyPodCount(selector, int(numPods)) env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration - // Set the expireAfter value to get the node deleted - nodePool.Spec.Disruption.ExpireAfter.Duration = lo.ToPtr(time.Minute) - env.ExpectUpdated(nodePool) - - env.EventuallyExpectExpired(nodeClaim) - - // Remove the do-not-disruption annotation so that the Nodes are now deprovisionable - for _, pod := range env.ExpectPodsMatchingSelector(selector) { - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) - env.ExpectUpdated(pod) - } - // Eventually the node will be tainted, which means its actively being disrupted Eventually(func(g Gomega) { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).Should(Succeed()) - _, ok := lo.Find(node.Spec.Taints, func(t v1.Taint) bool { - return corev1beta1.IsDisruptingTaint(t) + _, ok := lo.Find(node.Spec.Taints, func(t corev1.Taint) bool { + return karpv1.IsDisruptingTaint(t) }) g.Expect(ok).To(BeTrue()) }).Should(Succeed()) - // Set the expireAfter to "Never" to make sure new node isn't deleted - // This is CRITICAL since it prevents nodes that are immediately spun up from immediately being expired and - // racing at the end of the E2E test, leaking node resources into subsequent tests - nodePool.Spec.Disruption.ExpireAfter.Duration = nil + env.EventuallyExpectCreatedNodeCount("==", 2) + // Set the limit to 0 to make sure we don't continue to create nodeClaims. + // This is CRITICAL since it prevents leaking node resources into subsequent tests + nodePool.Spec.Limits = karpv1.Limits{ + corev1.ResourceCPU: resource.MustParse("0"), + } env.ExpectUpdated(nodePool) // After the deletion timestamp is set and all pods are drained @@ -572,165 +161,4 @@ var _ = Describe("Expiration", func() { env.EventuallyExpectCreatedNodeCount("==", 1) env.EventuallyExpectHealthyPodCount(selector, int(numPods)) }) - Context("Failure", func() { - It("should not continue to expire if a node never registers", func() { - // Launch a new NodeClaim - var numPods int32 = 2 - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: 2, - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "inflate"}}, - PodAntiRequirements: []v1.PodAffinityTerm{{ - TopologyKey: v1.LabelHostname, - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "inflate"}, - }}, - }, - }, - }) - env.ExpectCreated(dep, nodeClass, nodePool) - - startingNodeClaimState := env.EventuallyExpectCreatedNodeClaimCount("==", int(numPods)) - env.EventuallyExpectCreatedNodeCount("==", int(numPods)) - - // Set a configuration that will not register a NodeClaim - parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ - Name: lo.ToPtr("/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs"), - }) - Expect(err).ToNot(HaveOccurred()) - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: *parameter.Parameter.Value, - }, - } - env.ExpectCreatedOrUpdated(nodeClass) - - env.EventuallyExpectExpired(startingNodeClaimState...) - - // Expect nodes To get tainted - taintedNodes := env.EventuallyExpectTaintedNodeCount("==", 1) - - // Expire should fail and the original node should be untainted - // TODO: reduce timeouts when deprovisioning waits are factored out - env.EventuallyExpectNodesUntaintedWithTimeout(11*time.Minute, taintedNodes...) - - // The nodeclaims that never registers will be removed - Eventually(func(g Gomega) { - nodeClaims := &corev1beta1.NodeClaimList{} - g.Expect(env.Client.List(env, nodeClaims, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) - g.Expect(len(nodeClaims.Items)).To(BeNumerically("==", int(numPods))) - }).WithTimeout(6 * time.Minute).Should(Succeed()) - - // Expect all the NodeClaims that existed on the initial provisioning loop are not removed - Consistently(func(g Gomega) { - nodeClaims := &corev1beta1.NodeClaimList{} - g.Expect(env.Client.List(env, nodeClaims, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) - - startingNodeClaimUIDs := lo.Map(startingNodeClaimState, func(nc *corev1beta1.NodeClaim, _ int) types.UID { return nc.UID }) - nodeClaimUIDs := lo.Map(nodeClaims.Items, func(nc corev1beta1.NodeClaim, _ int) types.UID { return nc.UID }) - g.Expect(sets.New(nodeClaimUIDs...).IsSuperset(sets.New(startingNodeClaimUIDs...))).To(BeTrue()) - }, "2m").Should(Succeed()) - }) - It("should not continue to expire if a node registers but never becomes initialized", func() { - // Set a configuration that will allow us to make a NodeClaim not be initialized - nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com/taint", Effect: v1.TaintEffectPreferNoSchedule}} - - // Launch a new NodeClaim - var numPods int32 = 2 - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: 2, - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "inflate"}}, - PodAntiRequirements: []v1.PodAffinityTerm{{ - TopologyKey: v1.LabelHostname, - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "inflate"}, - }}, - }, - }, - }) - env.ExpectCreated(dep, nodeClass, nodePool) - - startingNodeClaimState := env.EventuallyExpectCreatedNodeClaimCount("==", int(numPods)) - nodes := env.EventuallyExpectCreatedNodeCount("==", int(numPods)) - - // Remove the startup taints from these nodes to initialize them - Eventually(func(g Gomega) { - for _, node := range nodes { - g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) - stored := node.DeepCopy() - node.Spec.Taints = lo.Reject(node.Spec.Taints, func(t v1.Taint, _ int) bool { return t.Key == "example.com/taint" }) - g.Expect(env.Client.Patch(env.Context, node, client.StrategicMergeFrom(stored))).To(Succeed()) - } - }).Should(Succeed()) - - env.EventuallyExpectExpired(startingNodeClaimState...) - - // Expect nodes To be tainted - taintedNodes := env.EventuallyExpectTaintedNodeCount("==", 1) - - // Expire should fail and original node should be untainted and no NodeClaims should be removed - // TODO: reduce timeouts when deprovisioning waits are factored out - env.EventuallyExpectNodesUntaintedWithTimeout(11*time.Minute, taintedNodes...) - - // Expect that the new NodeClaim/Node is kept around after the un-cordon - nodeList := &v1.NodeList{} - Expect(env.Client.List(env, nodeList, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) - Expect(nodeList.Items).To(HaveLen(int(numPods) + 1)) - - nodeClaimList := &corev1beta1.NodeClaimList{} - Expect(env.Client.List(env, nodeClaimList, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) - Expect(nodeClaimList.Items).To(HaveLen(int(numPods) + 1)) - - // Expect all the NodeClaims that existed on the initial provisioning loop are not removed - Consistently(func(g Gomega) { - nodeClaims := &corev1beta1.NodeClaimList{} - g.Expect(env.Client.List(env, nodeClaims, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) - - startingNodeClaimUIDs := lo.Map(startingNodeClaimState, func(nc *corev1beta1.NodeClaim, _ int) types.UID { return nc.UID }) - nodeClaimUIDs := lo.Map(nodeClaims.Items, func(nc corev1beta1.NodeClaim, _ int) types.UID { return nc.UID }) - g.Expect(sets.New(nodeClaimUIDs...).IsSuperset(sets.New(startingNodeClaimUIDs...))).To(BeTrue()) - }, "2m").Should(Succeed()) - }) - It("should not expire any nodes if their PodDisruptionBudgets are unhealthy", func() { - // Create a deployment that contains a readiness probe that will never succeed - // This way, the pod will bind to the node, but the PodDisruptionBudget will never go healthy - var numPods int32 = 2 - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: 2, - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "inflate"}}, - PodAntiRequirements: []v1.PodAffinityTerm{{ - TopologyKey: v1.LabelHostname, - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "inflate"}, - }}, - }, - ReadinessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - HTTPGet: &v1.HTTPGetAction{ - Port: intstr.FromInt32(80), - }, - }, - }, - }, - }) - selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) - minAvailable := intstr.FromInt32(numPods - 1) - pdb := coretest.PodDisruptionBudget(coretest.PDBOptions{ - Labels: dep.Spec.Template.Labels, - MinAvailable: &minAvailable, - }) - env.ExpectCreated(dep, nodeClass, nodePool, pdb) - - nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", int(numPods)) - env.EventuallyExpectCreatedNodeCount("==", int(numPods)) - - // Expect pods to be bound but not to be ready since we are intentionally failing the readiness check - env.EventuallyExpectBoundPodCount(selector, int(numPods)) - - env.EventuallyExpectExpired(nodeClaims...) - env.ConsistentlyExpectNoDisruptions(int(numPods), time.Minute) - }) - }) }) diff --git a/test/suites/integration/ami_test.go b/test/suites/integration/ami_test.go deleted file mode 100644 index 96c3bdc57a1c..000000000000 --- a/test/suites/integration/ami_test.go +++ /dev/null @@ -1,361 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package integration_test - -import ( - "encoding/base64" - "fmt" - "os" - "strings" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/ssm" - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - coretest "sigs.k8s.io/karpenter/pkg/test" - - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" - awsenv "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("AMI", func() { - var customAMI string - BeforeEach(func() { - customAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", 1) - }) - - It("should use the AMI defined by the AMI Selector Terms", func() { - pod := coretest.Pod() - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: customAMI, - }, - } - env.ExpectCreated(pod, nodeClass, nodePool) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - - env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) - }) - It("should use the most recent AMI when discovering multiple", func() { - // choose an old static image - oldCustomAMI := env.GetCustomAMI("/aws/service/eks/optimized-ami/%[1]s/amazon-linux-2023/x86_64/standard/amazon-eks-node-al2023-x86_64-standard-%[1]s-v20240307/image_id", 1) - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: customAMI, - }, - { - ID: oldCustomAMI, - }, - } - pod := coretest.Pod() - - env.ExpectCreated(pod, nodeClass, nodePool) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - - env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) - }) - It("should support AMI Selector Terms for Name but fail with incorrect owners", func() { - output, err := env.EC2API.DescribeImages(&ec2.DescribeImagesInput{ - ImageIds: []*string{aws.String(customAMI)}, - }) - Expect(err).To(BeNil()) - Expect(output.Images).To(HaveLen(1)) - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Name: *output.Images[0].Name, - Owner: "fakeOwnerValue", - }, - } - pod := coretest.Pod() - - env.ExpectCreated(pod, nodeClass, nodePool) - env.ExpectCreatedNodeCount("==", 0) - Expect(pod.Spec.NodeName).To(Equal("")) - }) - It("should support ami selector Name with default owners", func() { - output, err := env.EC2API.DescribeImages(&ec2.DescribeImagesInput{ - ImageIds: []*string{aws.String(customAMI)}, - }) - Expect(err).To(BeNil()) - Expect(output.Images).To(HaveLen(1)) - - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Name: *output.Images[0].Name, - }, - } - pod := coretest.Pod() - - env.ExpectCreated(pod, nodeClass, nodePool) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - - env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) - }) - It("should support ami selector ids", func() { - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: customAMI, - }, - } - pod := coretest.Pod() - - env.ExpectCreated(pod, nodeClass, nodePool) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - - env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) - }) - - Context("AMIFamily", func() { - It("should provision a node using the AL2 family", func() { - pod := coretest.Pod() - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 - env.ExpectCreated(nodeClass, nodePool, pod) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should provision a node using the AL2023 family", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 - pod := coretest.Pod() - env.ExpectCreated(nodeClass, nodePool, pod) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should provision a node using the Bottlerocket family", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket - pod := coretest.Pod() - env.ExpectCreated(nodeClass, nodePool, pod) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should provision a node using the Ubuntu family", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyUbuntu - // TODO (jmdeal@): remove once 22.04 AMIs are supported - if env.GetK8sVersion(0) == "1.29" { - nodeClass.Spec.AMISelectorTerms = lo.Map([]string{ - "/aws/service/canonical/ubuntu/eks/20.04/1.28/stable/current/amd64/hvm/ebs-gp2/ami-id", - "/aws/service/canonical/ubuntu/eks/20.04/1.28/stable/current/arm64/hvm/ebs-gp2/ami-id", - }, func(arg string, _ int) v1beta1.AMISelectorTerm { - parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{Name: lo.ToPtr(arg)}) - Expect(err).To(BeNil()) - return v1beta1.AMISelectorTerm{ID: *parameter.Parameter.Value} - }) - } - // TODO: remove requirements after Ubuntu fixes bootstrap script issue w/ - // new instance types not included in the max-pods.txt file. (https://github.com/aws/karpenter-provider-aws/issues/4472) - nodePool = coretest.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, - Values: awsenv.ExcludedInstanceFamilies, - }, - }, - ) - pod := coretest.Pod() - env.ExpectCreated(nodeClass, nodePool, pod) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support Custom AMIFamily with AMI Selectors", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom - al2AMI := env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: al2AMI, - }, - } - nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) - pod := coretest.Pod() - - env.ExpectCreated(pod, nodeClass, nodePool) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - - env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(al2AMI)))) - }) - It("should have the EC2NodeClass status for AMIs using wildcard", func() { - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Name: "*", - }, - } - env.ExpectCreated(nodeClass) - nc := EventuallyExpectAMIsToExist(nodeClass) - Expect(len(nc.Status.AMIs)).To(BeNumerically("<", 10)) - }) - It("should have the EC2NodeClass status for AMIs using tags", func() { - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: customAMI, - }, - } - env.ExpectCreated(nodeClass) - nc := EventuallyExpectAMIsToExist(nodeClass) - Expect(len(nc.Status.AMIs)).To(BeNumerically("==", 1)) - Expect(nc.Status.AMIs[0].ID).To(Equal(customAMI)) - }) - }) - - Context("UserData", func() { - It("should merge UserData contents for AL2 AMIFamily", func() { - content, err := os.ReadFile("testdata/al2_userdata_input.sh") - Expect(err).ToNot(HaveOccurred()) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 - nodeClass.Spec.UserData = aws.String(string(content)) - nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} - nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} - pod := coretest.Pod(coretest.PodOptions{Tolerations: []v1.Toleration{{Key: "example.com", Operator: v1.TolerationOpExists}}}) - - env.ExpectCreated(pod, nodeClass, nodePool) - env.EventuallyExpectHealthy(pod) - Expect(env.GetNode(pod.Spec.NodeName).Spec.Taints).To(ContainElements( - v1.Taint{Key: "example.com", Value: "value", Effect: "NoExecute"}, - v1.Taint{Key: "example.com", Value: "value", Effect: "NoSchedule"}, - )) - actualUserData, err := base64.StdEncoding.DecodeString(*getInstanceAttribute(pod.Spec.NodeName, "userData").UserData.Value) - Expect(err).ToNot(HaveOccurred()) - // Since the node has joined the cluster, we know our bootstrapping was correct. - // Just verify if the UserData contains our custom content too, rather than doing a byte-wise comparison. - Expect(string(actualUserData)).To(ContainSubstring("Running custom user data script")) - }) - It("should merge non-MIME UserData contents for AL2 AMIFamily", func() { - content, err := os.ReadFile("testdata/al2_no_mime_userdata_input.sh") - Expect(err).ToNot(HaveOccurred()) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 - nodeClass.Spec.UserData = aws.String(string(content)) - nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} - nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} - pod := coretest.Pod(coretest.PodOptions{Tolerations: []v1.Toleration{{Key: "example.com", Operator: v1.TolerationOpExists}}}) - - env.ExpectCreated(pod, nodeClass, nodePool) - env.EventuallyExpectHealthy(pod) - Expect(env.GetNode(pod.Spec.NodeName).Spec.Taints).To(ContainElements( - v1.Taint{Key: "example.com", Value: "value", Effect: "NoExecute"}, - v1.Taint{Key: "example.com", Value: "value", Effect: "NoSchedule"}, - )) - actualUserData, err := base64.StdEncoding.DecodeString(*getInstanceAttribute(pod.Spec.NodeName, "userData").UserData.Value) - Expect(err).ToNot(HaveOccurred()) - // Since the node has joined the cluster, we know our bootstrapping was correct. - // Just verify if the UserData contains our custom content too, rather than doing a byte-wise comparison. - Expect(string(actualUserData)).To(ContainSubstring("Running custom user data script")) - }) - It("should merge UserData contents for Bottlerocket AMIFamily", func() { - content, err := os.ReadFile("testdata/br_userdata_input.sh") - Expect(err).ToNot(HaveOccurred()) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket - nodeClass.Spec.UserData = aws.String(string(content)) - nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} - nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} - pod := coretest.Pod(coretest.PodOptions{Tolerations: []v1.Toleration{{Key: "example.com", Operator: v1.TolerationOpExists}}}) - - env.ExpectCreated(pod, nodeClass, nodePool) - env.EventuallyExpectHealthy(pod) - Expect(env.GetNode(pod.Spec.NodeName).Spec.Taints).To(ContainElements( - v1.Taint{Key: "example.com", Value: "value", Effect: "NoExecute"}, - v1.Taint{Key: "example.com", Value: "value", Effect: "NoSchedule"}, - )) - actualUserData, err := base64.StdEncoding.DecodeString(*getInstanceAttribute(pod.Spec.NodeName, "userData").UserData.Value) - Expect(err).ToNot(HaveOccurred()) - Expect(string(actualUserData)).To(ContainSubstring("kube-api-qps = 30")) - }) - It("should merge UserData contents for Windows AMIFamily", func() { - env.ExpectWindowsIPAMEnabled() - DeferCleanup(func() { - env.ExpectWindowsIPAMDisabled() - }) - - content, err := os.ReadFile("testdata/windows_userdata_input.ps1") - Expect(err).ToNot(HaveOccurred()) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 - nodeClass.Spec.UserData = aws.String(string(content)) - nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} - nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} - - // TODO: remove this requirement once VPC RC rolls out m7a.*, r7a.* ENI data (https://github.com/aws/karpenter-provider-aws/issues/4472) - nodePool = coretest.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, - Values: awsenv.ExcludedInstanceFamilies, - }, - }, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Windows)}, - }, - }, - ) - pod := coretest.Pod(coretest.PodOptions{ - Image: awsenv.WindowsDefaultImage, - NodeSelector: map[string]string{ - v1.LabelOSStable: string(v1.Windows), - v1.LabelWindowsBuild: "10.0.20348", - }, - Tolerations: []v1.Toleration{{Key: "example.com", Operator: v1.TolerationOpExists}}, - }) - - env.ExpectCreated(pod, nodeClass, nodePool) - env.EventuallyExpectHealthyWithTimeout(time.Minute*15, pod) // Wait 15 minutes because Windows nodes/containers take longer to spin up - Expect(env.GetNode(pod.Spec.NodeName).Spec.Taints).To(ContainElements( - v1.Taint{Key: "example.com", Value: "value", Effect: "NoExecute"}, - v1.Taint{Key: "example.com", Value: "value", Effect: "NoSchedule"}, - )) - actualUserData, err := base64.StdEncoding.DecodeString(*getInstanceAttribute(pod.Spec.NodeName, "userData").UserData.Value) - Expect(err).ToNot(HaveOccurred()) - Expect(string(actualUserData)).To(ContainSubstring("Write-Host \"Running custom user data script\"")) - Expect(string(actualUserData)).To(ContainSubstring("[string]$EKSBootstrapScriptFile = \"$env:ProgramFiles\\Amazon\\EKS\\Start-EKSBootstrap.ps1\"")) - }) - }) -}) - -//nolint:unparam -func getInstanceAttribute(nodeName string, attribute string) *ec2.DescribeInstanceAttributeOutput { - var node v1.Node - Expect(env.Client.Get(env.Context, types.NamespacedName{Name: nodeName}, &node)).To(Succeed()) - providerIDSplit := strings.Split(node.Spec.ProviderID, "/") - instanceID := providerIDSplit[len(providerIDSplit)-1] - instanceAttribute, err := env.EC2API.DescribeInstanceAttribute(&ec2.DescribeInstanceAttributeInput{ - InstanceId: aws.String(instanceID), - Attribute: aws.String(attribute), - }) - Expect(err).ToNot(HaveOccurred()) - return instanceAttribute -} - -func EventuallyExpectAMIsToExist(nodeClass *v1beta1.EC2NodeClass) *v1beta1.EC2NodeClass { - nc := &v1beta1.EC2NodeClass{} - Eventually(func(g Gomega) { - g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClass), nc)).To(Succeed()) - g.Expect(nc.Status.AMIs).ToNot(BeNil()) - }).WithTimeout(30 * time.Second).Should(Succeed()) - return nc -} diff --git a/test/suites/integration/aws_metadata_test.go b/test/suites/integration/aws_metadata_test.go index a13d5ad30261..d48600206bc7 100644 --- a/test/suites/integration/aws_metadata_test.go +++ b/test/suites/integration/aws_metadata_test.go @@ -19,7 +19,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" coretest "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -27,7 +27,7 @@ import ( var _ = Describe("MetadataOptions", func() { It("should use specified metadata options", func() { - nodeClass.Spec.MetadataOptions = &v1beta1.MetadataOptions{ + nodeClass.Spec.MetadataOptions = &v1.MetadataOptions{ HTTPEndpoint: aws.String("enabled"), HTTPProtocolIPv6: aws.String("enabled"), HTTPPutResponseHopLimit: aws.Int64(1), diff --git a/test/suites/integration/block_device_mappings_test.go b/test/suites/integration/block_device_mappings_test.go index 56e0643eeaa0..0006b1fa9431 100644 --- a/test/suites/integration/block_device_mappings_test.go +++ b/test/suites/integration/block_device_mappings_test.go @@ -19,7 +19,7 @@ import ( "sigs.k8s.io/karpenter/pkg/test" "sigs.k8s.io/karpenter/pkg/utils/resources" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -27,10 +27,10 @@ import ( var _ = Describe("BlockDeviceMappings", func() { It("should use specified block device mappings", func() { - nodeClass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ + nodeClass.Spec.BlockDeviceMappings = []*v1.BlockDeviceMapping{ { DeviceName: aws.String("/dev/xvda"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ VolumeSize: resources.Quantity("20Gi"), VolumeType: aws.String("io2"), IOPS: aws.Int64(1000), diff --git a/test/suites/integration/cni_test.go b/test/suites/integration/cni_test.go index 97ce0999001a..41d65f04dae3 100644 --- a/test/suites/integration/cni_test.go +++ b/test/suites/integration/cni_test.go @@ -25,7 +25,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" ) @@ -42,7 +42,7 @@ var _ = Describe("CNITests", func() { Expect(allocatablePods).To(Equal(eniLimitedPodsFor(node.Labels["node.kubernetes.io/instance-type"]))) }) It("should set max pods to 110 if maxPods is set in kubelet", func() { - nodePool.Spec.Template.Spec.Kubelet = &v1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} pod := test.Pod() env.ExpectCreated(pod, nodeClass, nodePool) env.EventuallyExpectHealthy(pod) diff --git a/test/suites/integration/conversion_webhook_test.go b/test/suites/integration/conversion_webhook_test.go new file mode 100644 index 000000000000..544285a65038 --- /dev/null +++ b/test/suites/integration/conversion_webhook_test.go @@ -0,0 +1,239 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karptest "sigs.k8s.io/karpenter/pkg/test" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/pkg/test" +) + +var _ = Describe("Conversion Webhooks", func() { + Context("NodePool", func() { + It("should not update a metadata generation when the same resource is applied for the v1beta1 APIs", func() { + // created v1beta1 resource + storedv1beta1NodePool := &karpv1beta1.NodePool{ + ObjectMeta: karptest.ObjectMeta(), + Spec: karpv1beta1.NodePoolSpec{ + Limits: karpv1beta1.Limits{ + corev1.ResourceCPU: lo.Must(resource.ParseQuantity("20m")), + }, + Disruption: karpv1beta1.Disruption{ + ConsolidationPolicy: karpv1beta1.ConsolidationPolicyWhenEmpty, + ConsolidateAfter: lo.ToPtr(karpv1beta1.MustParseNillableDuration("1h")), + ExpireAfter: karpv1beta1.MustParseNillableDuration("1h"), + }, + Template: karpv1beta1.NodeClaimTemplate{ + Spec: karpv1beta1.NodeClaimSpec{ + NodeClassRef: &karpv1beta1.NodeClassReference{ + Name: "test-nodeclass", + }, + Requirements: []karpv1beta1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Linux)}, + }, + }, + }, + Kubelet: &karpv1beta1.KubeletConfiguration{ + MaxPods: lo.ToPtr(int32(110)), + PodsPerCore: lo.ToPtr(int32(10)), + SystemReserved: map[string]string{ + string(corev1.ResourceCPU): "200m", + string(corev1.ResourceMemory): "200Mi", + string(corev1.ResourceEphemeralStorage): "1Gi", + }, + KubeReserved: map[string]string{ + string(corev1.ResourceCPU): "200m", + string(corev1.ResourceMemory): "200Mi", + string(corev1.ResourceEphemeralStorage): "1Gi", + }, + EvictionHard: map[string]string{ + "memory.available": "5%", + "nodefs.available": "5%", + "nodefs.inodesFree": "5%", + "imagefs.available": "5%", + "imagefs.inodesFree": "5%", + "pid.available": "3%", + }, + EvictionSoft: map[string]string{ + "memory.available": "10%", + "nodefs.available": "10%", + "nodefs.inodesFree": "10%", + "imagefs.available": "10%", + "imagefs.inodesFree": "10%", + "pid.available": "6%", + }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "memory.available": {Duration: time.Minute * 2}, + "nodefs.available": {Duration: time.Minute * 2}, + "nodefs.inodesFree": {Duration: time.Minute * 2}, + "imagefs.available": {Duration: time.Minute * 2}, + "imagefs.inodesFree": {Duration: time.Minute * 2}, + "pid.available": {Duration: time.Minute * 2}, + }, + EvictionMaxPodGracePeriod: lo.ToPtr(int32(120)), + ImageGCHighThresholdPercent: lo.ToPtr(int32(50)), + ImageGCLowThresholdPercent: lo.ToPtr(int32(10)), + CPUCFSQuota: lo.ToPtr(false), + }, + }, + }, + }, + } + + // Use a deepcopy to make sure the nodePool object is not populated with the returned object from the APIServer + env.ExpectCreated(storedv1beta1NodePool.DeepCopy()) + v1beta1NodePool := env.ExpectExists(storedv1beta1NodePool.DeepCopy()).(*karpv1beta1.NodePool) + Expect(v1beta1NodePool.Generation).To(BeNumerically("==", 1)) + + // Second apply of the same NodePool does not increase the generation + env.ExpectUpdated(storedv1beta1NodePool.DeepCopy()) + v1beta1NodePool = env.ExpectExists(storedv1beta1NodePool.DeepCopy()).(*karpv1beta1.NodePool) + Expect(v1beta1NodePool.Generation).To(BeNumerically("==", 1)) + Expect(v1beta1NodePool.Spec.Template.Spec.NodeClassRef.APIVersion).To(Equal(storedv1beta1NodePool.Spec.Template.Spec.NodeClassRef.APIVersion)) + Expect(v1beta1NodePool.Spec.Disruption.ConsolidateAfter.Duration.String()).To(Equal(storedv1beta1NodePool.Spec.Disruption.ConsolidateAfter.Duration.String())) + Expect(v1beta1NodePool.Spec.Disruption.ExpireAfter.Duration.String()).To(Equal(storedv1beta1NodePool.Spec.Disruption.ExpireAfter.Duration.String())) + // Kubelet Validation + Expect(v1beta1NodePool.Spec.Template.Spec.Kubelet.MaxPods).To(Equal(storedv1beta1NodePool.Spec.Template.Spec.Kubelet.MaxPods)) + Expect(v1beta1NodePool.Spec.Template.Spec.Kubelet.PodsPerCore).To(Equal(storedv1beta1NodePool.Spec.Template.Spec.Kubelet.PodsPerCore)) + Expect(v1beta1NodePool.Spec.Template.Spec.Kubelet.SystemReserved).To(Equal(storedv1beta1NodePool.Spec.Template.Spec.Kubelet.SystemReserved)) + Expect(v1beta1NodePool.Spec.Template.Spec.Kubelet.KubeReserved).To(Equal(storedv1beta1NodePool.Spec.Template.Spec.Kubelet.KubeReserved)) + Expect(v1beta1NodePool.Spec.Template.Spec.Kubelet.EvictionHard).To(Equal(storedv1beta1NodePool.Spec.Template.Spec.Kubelet.EvictionHard)) + Expect(v1beta1NodePool.Spec.Template.Spec.Kubelet.EvictionSoft).To(Equal(storedv1beta1NodePool.Spec.Template.Spec.Kubelet.EvictionSoft)) + Expect(v1beta1NodePool.Spec.Template.Spec.Kubelet.EvictionSoftGracePeriod).To(Equal(storedv1beta1NodePool.Spec.Template.Spec.Kubelet.EvictionSoftGracePeriod)) + Expect(v1beta1NodePool.Spec.Template.Spec.Kubelet.EvictionMaxPodGracePeriod).To(Equal(storedv1beta1NodePool.Spec.Template.Spec.Kubelet.EvictionMaxPodGracePeriod)) + Expect(v1beta1NodePool.Spec.Template.Spec.Kubelet.ImageGCHighThresholdPercent).To(Equal(storedv1beta1NodePool.Spec.Template.Spec.Kubelet.ImageGCHighThresholdPercent)) + Expect(v1beta1NodePool.Spec.Template.Spec.Kubelet.ImageGCLowThresholdPercent).To(Equal(storedv1beta1NodePool.Spec.Template.Spec.Kubelet.ImageGCLowThresholdPercent)) + Expect(v1beta1NodePool.Spec.Template.Spec.Kubelet.CPUCFSQuota).To(Equal(storedv1beta1NodePool.Spec.Template.Spec.Kubelet.CPUCFSQuota)) + }) + It("should not update a metadata generation when the same resource is applied for the v1 APIs", func() { + nodePool.Spec.Disruption = karpv1.Disruption{ + ConsolidateAfter: karpv1.MustParseNillableDuration("1h"), + } + nodePool.Spec.Template.Spec.ExpireAfter = karpv1.MustParseNillableDuration("1h") + nodePool.Spec.Limits = karpv1.Limits{ + corev1.ResourceCPU: lo.Must(resource.ParseQuantity("20m")), + } + + // Use a deepcopy to make sure the nodePool object is not populated with the returned object from the APIServer + env.ExpectCreated(nodePool.DeepCopy()) + v1NodePool := env.ExpectExists(nodePool).(*karpv1.NodePool) + Expect(v1NodePool.Generation).To(BeNumerically("==", 1)) + + // Second apply of the same NodePool does not increase the generation + env.ExpectUpdated(nodePool.DeepCopy()) + v1NodePool = env.ExpectExists(nodePool).(*karpv1.NodePool) + Expect(v1NodePool.Generation).To(BeNumerically("==", 1)) + Expect(v1NodePool.Spec.Template.Spec.NodeClassRef.Group).To(Equal(nodePool.Spec.Template.Spec.NodeClassRef.Group)) + Expect(v1NodePool.Spec.Disruption.ConsolidateAfter.Duration.String()).To(Equal(nodePool.Spec.Disruption.ConsolidateAfter.Duration.String())) + Expect(v1NodePool.Spec.Template.Spec.ExpireAfter.Duration.String()).To(Equal(nodePool.Spec.Template.Spec.ExpireAfter.Duration.String())) + }) + }) + Context("EC2NodeClass", func() { + It("should not update a metadata generation when the same resource is applied for the v1beta1 APIs", func() { + // created v1beta1 resource + storedv1beta1nodeclass := test.BetaEC2NodeClass() + + env.ExpectCreated(storedv1beta1nodeclass.DeepCopy()) + v1beta1nodeclass := env.ExpectExists(storedv1beta1nodeclass.DeepCopy()).(*v1beta1.EC2NodeClass) + Expect(v1beta1nodeclass.Generation).To(BeNumerically("==", 1)) + + // Second apply of the same NodeClass does not increase the generation + env.ExpectUpdated(storedv1beta1nodeclass.DeepCopy()) + v1beta1nodeclass = env.ExpectExists(storedv1beta1nodeclass).(*v1beta1.EC2NodeClass) + Expect(v1beta1nodeclass.Generation).To(BeNumerically("==", 1)) + }) + It("should not update a metadata generation when the same resource is applied for v1 APIs", func() { + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + MaxPods: lo.ToPtr(int32(110)), + PodsPerCore: lo.ToPtr(int32(10)), + SystemReserved: map[string]string{ + string(corev1.ResourceCPU): "200m", + string(corev1.ResourceMemory): "200Mi", + string(corev1.ResourceEphemeralStorage): "1Gi", + }, + KubeReserved: map[string]string{ + string(corev1.ResourceCPU): "200m", + string(corev1.ResourceMemory): "200Mi", + string(corev1.ResourceEphemeralStorage): "1Gi", + }, + EvictionHard: map[string]string{ + "memory.available": "5%", + "nodefs.available": "5%", + "nodefs.inodesFree": "5%", + "imagefs.available": "5%", + "imagefs.inodesFree": "5%", + "pid.available": "3%", + }, + EvictionSoft: map[string]string{ + "memory.available": "10%", + "nodefs.available": "10%", + "nodefs.inodesFree": "10%", + "imagefs.available": "10%", + "imagefs.inodesFree": "10%", + "pid.available": "6%", + }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "memory.available": {Duration: time.Minute * 2}, + "nodefs.available": {Duration: time.Minute * 2}, + "nodefs.inodesFree": {Duration: time.Minute * 2}, + "imagefs.available": {Duration: time.Minute * 2}, + "imagefs.inodesFree": {Duration: time.Minute * 2}, + "pid.available": {Duration: time.Minute * 2}, + }, + EvictionMaxPodGracePeriod: lo.ToPtr(int32(120)), + ImageGCHighThresholdPercent: lo.ToPtr(int32(50)), + ImageGCLowThresholdPercent: lo.ToPtr(int32(10)), + CPUCFSQuota: lo.ToPtr(false), + } + // Use a deepcopy to make sure the nodePool object is not populated with the returned object from the APIServer + env.ExpectCreated(nodeClass.DeepCopy()) + v1nodeclass := env.ExpectExists(nodeClass.DeepCopy()).(*v1.EC2NodeClass) + Expect(v1nodeclass.Generation).To(BeNumerically("==", 1)) + + // Second apply of the same NodeClass does not increase the generation + env.ExpectUpdated(nodeClass.DeepCopy()) + v1nodeclass = env.ExpectExists(nodeClass.DeepCopy()).(*v1.EC2NodeClass) + Expect(v1nodeclass.Generation).To(BeNumerically("==", 1)) + // Kubelet Validation + Expect(v1nodeclass.Spec.Kubelet.MaxPods).To(Equal(nodeClass.Spec.Kubelet.MaxPods)) + Expect(v1nodeclass.Spec.Kubelet.PodsPerCore).To(Equal(nodeClass.Spec.Kubelet.PodsPerCore)) + Expect(v1nodeclass.Spec.Kubelet.SystemReserved).To(Equal(nodeClass.Spec.Kubelet.SystemReserved)) + Expect(v1nodeclass.Spec.Kubelet.KubeReserved).To(Equal(nodeClass.Spec.Kubelet.KubeReserved)) + Expect(v1nodeclass.Spec.Kubelet.EvictionHard).To(Equal(nodeClass.Spec.Kubelet.EvictionHard)) + Expect(v1nodeclass.Spec.Kubelet.EvictionSoft).To(Equal(nodeClass.Spec.Kubelet.EvictionSoft)) + Expect(v1nodeclass.Spec.Kubelet.EvictionSoftGracePeriod).To(Equal(nodeClass.Spec.Kubelet.EvictionSoftGracePeriod)) + Expect(v1nodeclass.Spec.Kubelet.EvictionMaxPodGracePeriod).To(Equal(nodeClass.Spec.Kubelet.EvictionMaxPodGracePeriod)) + Expect(v1nodeclass.Spec.Kubelet.ImageGCHighThresholdPercent).To(Equal(nodeClass.Spec.Kubelet.ImageGCHighThresholdPercent)) + Expect(v1nodeclass.Spec.Kubelet.ImageGCLowThresholdPercent).To(Equal(nodeClass.Spec.Kubelet.ImageGCLowThresholdPercent)) + Expect(v1nodeclass.Spec.Kubelet.CPUCFSQuota).To(Equal(nodeClass.Spec.Kubelet.CPUCFSQuota)) + }) + }) +}) diff --git a/test/suites/integration/daemonset_test.go b/test/suites/integration/daemonset_test.go index 98c7a0013d06..a617cf4a0876 100644 --- a/test/suites/integration/daemonset_test.go +++ b/test/suites/integration/daemonset_test.go @@ -16,7 +16,7 @@ package integration_test import ( appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" schedulingv1 "k8s.io/api/scheduling/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,19 +26,19 @@ import ( . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/client" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" ) var _ = Describe("DaemonSet", func() { - var limitrange *v1.LimitRange + var limitrange *corev1.LimitRange var priorityclass *schedulingv1.PriorityClass var daemonset *appsv1.DaemonSet var dep *appsv1.Deployment BeforeEach(func() { - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenUnderutilized - nodePool.Spec.Disruption.ConsolidateAfter = nil + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmptyOrUnderutilized + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") priorityclass = &schedulingv1.PriorityClass{ ObjectMeta: metav1.ObjectMeta{ Name: "high-priority-daemonsets", @@ -47,7 +47,7 @@ var _ = Describe("DaemonSet", func() { GlobalDefault: false, Description: "This priority class should be used for daemonsets.", } - limitrange = &v1.LimitRange{ + limitrange = &corev1.LimitRange{ ObjectMeta: metav1.ObjectMeta{ Name: "limitrange", Namespace: "default", @@ -55,7 +55,7 @@ var _ = Describe("DaemonSet", func() { } daemonset = test.DaemonSet(test.DaemonSetOptions{ PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Limits: v1.ResourceList{v1.ResourceMemory: resource.MustParse("1Gi")}}, + ResourceRequirements: corev1.ResourceRequirements{Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}}, PriorityClassName: "high-priority-daemonsets", }, }) @@ -66,19 +66,19 @@ var _ = Describe("DaemonSet", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "large-app"}, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceMemory: resource.MustParse("4")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4")}, }, }, }) }) It("should account for LimitRange Default on daemonSet pods for resources", func() { - limitrange.Spec.Limits = []v1.LimitRangeItem{ + limitrange.Spec.Limits = []corev1.LimitRangeItem{ { - Type: v1.LimitTypeContainer, - Default: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("1Gi"), + Type: corev1.LimitTypeContainer, + Default: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("1Gi"), }, }, } @@ -89,7 +89,7 @@ var _ = Describe("DaemonSet", func() { // Eventually expect a single node to exist and both the deployment pod and the daemonset pod to schedule to it Eventually(func(g Gomega) { - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} g.Expect(env.Client.List(env, nodeList, client.HasLabels{"testing/cluster"})).To(Succeed()) g.Expect(nodeList.Items).To(HaveLen(1)) @@ -104,12 +104,12 @@ var _ = Describe("DaemonSet", func() { }).Should(Succeed()) }) It("should account for LimitRange DefaultRequest on daemonSet pods for resources", func() { - limitrange.Spec.Limits = []v1.LimitRangeItem{ + limitrange.Spec.Limits = []corev1.LimitRangeItem{ { - Type: v1.LimitTypeContainer, - DefaultRequest: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("1Gi"), + Type: corev1.LimitTypeContainer, + DefaultRequest: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("1Gi"), }, }, } @@ -120,7 +120,7 @@ var _ = Describe("DaemonSet", func() { // Eventually expect a single node to exist and both the deployment pod and the daemonset pod to schedule to it Eventually(func(g Gomega) { - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} g.Expect(env.Client.List(env, nodeList, client.HasLabels{"testing/cluster"})).To(Succeed()) g.Expect(nodeList.Items).To(HaveLen(1)) diff --git a/test/suites/integration/ec2nodeclass_kubelet_test.go b/test/suites/integration/ec2nodeclass_kubelet_test.go new file mode 100644 index 000000000000..5b48c0f2474a --- /dev/null +++ b/test/suites/integration/ec2nodeclass_kubelet_test.go @@ -0,0 +1,175 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration_test + +import ( + "fmt" + "strings" + "time" + + "github.com/awslabs/operatorpkg/object" + coretest "sigs.k8s.io/karpenter/pkg/test" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/pkg/test" + + "github.com/Pallinder/go-randomdata" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" +) + +var _ = Describe("EC2nodeClass Kubelet Configuration", func() { + var betaNodePool *karpv1beta1.NodePool + var betaNodeClass *v1beta1.EC2NodeClass + BeforeEach(func() { + betaNodeClass = test.BetaEC2NodeClass(v1beta1.EC2NodeClass{ + Spec: v1beta1.EC2NodeClassSpec{ + AMIFamily: lo.ToPtr(v1beta1.AMIFamilyAL2023), + Role: fmt.Sprintf("KarpenterNodeRole-%s", env.ClusterName), + Tags: map[string]string{ + "testing/cluster": env.ClusterName, + }, + SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + }, + }, + SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + }, + }, + }, + }) + betaNodePool = &karpv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, + Spec: karpv1beta1.NodePoolSpec{ + Template: karpv1beta1.NodeClaimTemplate{ + Spec: karpv1beta1.NodeClaimSpec{ + NodeClassRef: &karpv1beta1.NodeClassReference{ + APIVersion: object.GVK(betaNodeClass).GroupVersion().String(), + Kind: object.GVK(betaNodeClass).Kind, + Name: betaNodeClass.Name, + }, + Requirements: []karpv1beta1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Linux)}, + }, + }, + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, + }, + }, + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"c", "m", "r"}, + }, + }, + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceGeneration, + Operator: corev1.NodeSelectorOpGt, + Values: []string{"2"}, + }, + }, + // Filter out a1 instance types, which are incompatible with AL2023 AMIs + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceFamily, + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"a1"}, + }, + }, + }, + Kubelet: &karpv1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](20)}, + }, + }, + }, + } + }) + It("should expect v1beta1 nodePool to have kubelet compatibility annotation", func() { + env.ExpectCreated(betaNodeClass, betaNodePool) + Eventually(func(g Gomega) { + np := &karpv1.NodePool{} + expectExists(betaNodePool, np) + + hash, found := np.Annotations[karpv1.KubeletCompatibilityAnnotationKey] + g.Expect(found).To(BeTrue()) + g.Expect(hash).To(Equal(np.Hash())) + }) + }) + It("should expect nodeClaim to not drift when kubelet configuration on v1 nodeClass is same as v1beta1 nodePool", func() { + pod := coretest.Pod() + env.ExpectCreated(pod, betaNodeClass, betaNodePool) + env.EventuallyExpectHealthy(pod) + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + _, found := nodeClaim.Annotations[v1.AnnotationKubeletCompatibilityHash] + Expect(found).To(BeTrue()) + + np := &karpv1.NodePool{} + expectExists(betaNodePool, np) + nc := &v1.EC2NodeClass{} + expectExists(betaNodeClass, nc) + + delete(np.Annotations, karpv1.KubeletCompatibilityAnnotationKey) + nc.Spec.Kubelet = &v1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](20)} + env.ExpectUpdated(np, nc) + + Eventually(func(g Gomega) { + g.Expect(nodeClaim.StatusConditions().Get(karpv1.ConditionTypeDrifted).IsUnknown()).To(BeTrue()) + }).Should(Succeed()) + }) + It("should expect nodeClaim to drift when kubelet configuration on v1 nodeClass is different from v1beta1 nodePool", func() { + pod := coretest.Pod() + env.ExpectCreated(pod, betaNodeClass, betaNodePool) + env.EventuallyExpectHealthy(pod) + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + _, found := nodeClaim.Annotations[v1.AnnotationKubeletCompatibilityHash] + Expect(found).To(BeTrue()) + + np := &karpv1.NodePool{} + expectExists(betaNodePool, np) + nc := &v1.EC2NodeClass{} + expectExists(betaNodeClass, nc) + + delete(np.Annotations, karpv1.KubeletCompatibilityAnnotationKey) + nc.Spec.Kubelet = &v1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](10)} + env.ExpectUpdated(np, nc) + env.EventuallyExpectDrifted(nodeClaim) + }) +}) + +func expectExists(v1beta1object client.Object, v1object client.Object) { + GinkgoHelper() + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(v1beta1object), v1object)).To(Succeed()) + }).WithTimeout(time.Second * 5).Should(Succeed()) +} diff --git a/test/suites/integration/extended_resources_test.go b/test/suites/integration/extended_resources_test.go index 49c2e8d3c7ae..48c1ad0116fe 100644 --- a/test/suites/integration/extended_resources_test.go +++ b/test/suites/integration/extended_resources_test.go @@ -21,20 +21,19 @@ import ( "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/karpenter/pkg/test" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" - awsenv "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" ) var _ = Describe("Extended Resources", func() { @@ -46,7 +45,7 @@ var _ = Describe("Extended Resources", func() { It("should provision nodes for a deployment that requests nvidia.com/gpu", func() { ExpectNvidiaDevicePluginCreated() // TODO: jmdeal@ remove AL2 pin once AL2023 accelerated AMIs are available - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} numPods := 1 dep := test.Deployment(test.DeploymentOptions{ Replicas: int32(numPods), @@ -54,21 +53,21 @@ var _ = Describe("Extended Resources", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "large-app"}, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ "nvidia.com/gpu": resource.MustParse("1"), }, - Limits: v1.ResourceList{ + Limits: corev1.ResourceList{ "nvidia.com/gpu": resource.MustParse("1"), }, }, }, }) selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpExists, + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpExists, }, }) env.ExpectCreated(nodeClass, nodePool, dep) @@ -78,7 +77,7 @@ var _ = Describe("Extended Resources", func() { }) It("should provision nodes for a deployment that requests nvidia.com/gpu (Bottlerocket)", func() { // For Bottlerocket, we are testing that resources are initialized without needing a device plugin - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} numPods := 1 dep := test.Deployment(test.DeploymentOptions{ Replicas: int32(numPods), @@ -86,21 +85,21 @@ var _ = Describe("Extended Resources", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "large-app"}, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ "nvidia.com/gpu": resource.MustParse("1"), }, - Limits: v1.ResourceList{ + Limits: corev1.ResourceList{ "nvidia.com/gpu": resource.MustParse("1"), }, }, }, }) selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpExists, + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpExists, }}) env.ExpectCreated(nodeClass, nodePool, dep) env.EventuallyExpectHealthyPodCount(selector, numPods) @@ -112,16 +111,6 @@ var _ = Describe("Extended Resources", func() { DeferCleanup(func() { env.ExpectPodENIDisabled() }) - // TODO: remove this requirement once VPC RC rolls out m7a.*, r7a.* ENI data (https://github.com/aws/karpenter-provider-aws/issues/4472) - test.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, - Values: awsenv.ExcludedInstanceFamilies, - }, - }, - ) numPods := 1 dep := test.Deployment(test.DeploymentOptions{ Replicas: int32(numPods), @@ -129,11 +118,11 @@ var _ = Describe("Extended Resources", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "large-app"}, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ "vpc.amazonaws.com/pod-eni": resource.MustParse("1"), }, - Limits: v1.ResourceList{ + Limits: corev1.ResourceList{ "vpc.amazonaws.com/pod-eni": resource.MustParse("1"), }, }, @@ -149,18 +138,14 @@ var _ = Describe("Extended Resources", func() { Skip("skipping test on AMD instance types") ExpectAMDDevicePluginCreated() - customAMI := env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 0) + customAMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersion())) // We create custom userData that installs the AMD Radeon driver and then performs the EKS bootstrap script // We use a Custom AMI so that we can reboot after we start the kubelet service rawContent, err := os.ReadFile("testdata/amd_driver_input.sh") Expect(err).ToNot(HaveOccurred()) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: customAMI, - }, - } + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: customAMI}} nodeClass.Spec.UserData = lo.ToPtr(fmt.Sprintf(string(rawContent), env.ClusterName, env.ClusterEndpoint, env.ExpectCABundle(), nodePool.Name)) @@ -171,11 +156,11 @@ var _ = Describe("Extended Resources", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "large-app"}, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ "amd.com/gpu": resource.MustParse("1"), }, - Limits: v1.ResourceList{ + Limits: corev1.ResourceList{ "amd.com/gpu": resource.MustParse("1"), }, }, @@ -195,7 +180,8 @@ var _ = Describe("Extended Resources", func() { Skip("skipping test on an exotic instance type") ExpectHabanaDevicePluginCreated() - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ { ID: "ami-0fae925f94979981f", }, @@ -207,11 +193,11 @@ var _ = Describe("Extended Resources", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "large-app"}, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ "habana.ai/gaudi": resource.MustParse("1"), }, - Limits: v1.ResourceList{ + Limits: corev1.ResourceList{ "habana.ai/gaudi": resource.MustParse("1"), }, }, @@ -230,16 +216,16 @@ var _ = Describe("Extended Resources", func() { nodePool.Spec.Template.Labels = map[string]string{ "aws.amazon.com/efa": "true", } - nodePool.Spec.Template.Spec.Taints = []v1.Taint{ + nodePool.Spec.Template.Spec.Taints = []corev1.Taint{ { Key: "aws.amazon.com/efa", - Effect: v1.TaintEffectNoSchedule, + Effect: corev1.TaintEffectNoSchedule, }, } // Only select private subnets since instances with multiple network instances at launch won't get a public IP. nodeClass.Spec.SubnetSelectorTerms[0].Tags["Name"] = "*Private*" // TODO: jmdeal@ remove AL2 pin once AL2023 accelerated AMIs are available - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} numPods := 1 dep := test.Deployment(test.DeploymentOptions{ @@ -248,17 +234,17 @@ var _ = Describe("Extended Resources", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "efa-app"}, }, - Tolerations: []v1.Toleration{ + Tolerations: []corev1.Toleration{ { Key: "aws.amazon.com/efa", - Operator: v1.TolerationOpExists, + Operator: corev1.TolerationOpExists, }, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ "vpc.amazonaws.com/efa": resource.MustParse("2"), }, - Limits: v1.ResourceList{ + Limits: corev1.ResourceList{ "vpc.amazonaws.com/efa": resource.MustParse("2"), }, }, @@ -288,38 +274,38 @@ func ExpectNvidiaDevicePluginCreated() { UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ Type: appsv1.RollingUpdateDaemonSetStrategyType, }, - Template: v1.PodTemplateSpec{ + Template: corev1.PodTemplateSpec{ ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{ Labels: map[string]string{ "name": "nvidia-device-plugin-ds", }, }), - Spec: v1.PodSpec{ - Tolerations: []v1.Toleration{ + Spec: corev1.PodSpec{ + Tolerations: []corev1.Toleration{ { Key: "nvidia.com/gpu", - Operator: v1.TolerationOpExists, - Effect: v1.TaintEffectNoSchedule, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, }, }, PriorityClassName: "system-node-critical", - Containers: []v1.Container{ + Containers: []corev1.Container{ { Name: "nvidia-device-plugin-ctr", Image: "nvcr.io/nvidia/k8s-device-plugin:v0.12.3", - Env: []v1.EnvVar{ + Env: []corev1.EnvVar{ { Name: "FAIL_ON_INIT_ERROR", Value: "false", }, }, - SecurityContext: &v1.SecurityContext{ + SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: lo.ToPtr(false), - Capabilities: &v1.Capabilities{ - Drop: []v1.Capability{"ALL"}, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, }, }, - VolumeMounts: []v1.VolumeMount{ + VolumeMounts: []corev1.VolumeMount{ { Name: "device-plugin", MountPath: "/var/lib/kubelet/device-plugins", @@ -327,11 +313,11 @@ func ExpectNvidiaDevicePluginCreated() { }, }, }, - Volumes: []v1.Volume{ + Volumes: []corev1.Volume{ { Name: "device-plugin", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ Path: "/var/lib/kubelet/device-plugins", }, }, @@ -356,32 +342,32 @@ func ExpectAMDDevicePluginCreated() { "name": "amdgpu-dp-ds", }, }, - Template: v1.PodTemplateSpec{ + Template: corev1.PodTemplateSpec{ ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{ Labels: map[string]string{ "name": "amdgpu-dp-ds", }, }), - Spec: v1.PodSpec{ + Spec: corev1.PodSpec{ PriorityClassName: "system-node-critical", - Tolerations: []v1.Toleration{ + Tolerations: []corev1.Toleration{ { Key: "amd.com/gpu", - Operator: v1.TolerationOpExists, - Effect: v1.TaintEffectNoSchedule, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, }, }, - Containers: []v1.Container{ + Containers: []corev1.Container{ { Name: "amdgpu-dp-cntr", Image: "rocm/k8s-device-plugin", - SecurityContext: &v1.SecurityContext{ + SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: lo.ToPtr(false), - Capabilities: &v1.Capabilities{ - Drop: []v1.Capability{"ALL"}, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, }, }, - VolumeMounts: []v1.VolumeMount{ + VolumeMounts: []corev1.VolumeMount{ { Name: "dp", MountPath: "/var/lib/kubelet/device-plugins", @@ -393,19 +379,19 @@ func ExpectAMDDevicePluginCreated() { }, }, }, - Volumes: []v1.Volume{ + Volumes: []corev1.Volume{ { Name: "dp", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ Path: "/var/lib/kubelet/device-plugins", }, }, }, { Name: "sys", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ Path: "/sys", }, }, @@ -419,7 +405,7 @@ func ExpectAMDDevicePluginCreated() { func ExpectHabanaDevicePluginCreated() { GinkgoHelper() - env.ExpectCreated(&v1.Namespace{ + env.ExpectCreated(&corev1.Namespace{ ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{ Name: "habana-system", }), @@ -438,7 +424,7 @@ func ExpectHabanaDevicePluginCreated() { UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ Type: appsv1.RollingUpdateDaemonSetStrategyType, }, - Template: v1.PodTemplateSpec{ + Template: corev1.PodTemplateSpec{ ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{ Annotations: map[string]string{ "scheduler.alpha.kubernetes.io/critical-pod": "", @@ -447,23 +433,23 @@ func ExpectHabanaDevicePluginCreated() { "name": "habanalabs-device-plugin-ds", }, }), - Spec: v1.PodSpec{ - Tolerations: []v1.Toleration{ + Spec: corev1.PodSpec{ + Tolerations: []corev1.Toleration{ { Key: "habana.ai/gaudi", - Operator: v1.TolerationOpExists, - Effect: v1.TaintEffectNoSchedule, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, }, }, PriorityClassName: "system-node-critical", - Containers: []v1.Container{ + Containers: []corev1.Container{ { Name: "habanalabs-device-plugin-ctr", Image: "vault.habana.ai/docker-k8s-device-plugin/docker-k8s-device-plugin:latest", - SecurityContext: &v1.SecurityContext{ + SecurityContext: &corev1.SecurityContext{ Privileged: lo.ToPtr(true), }, - VolumeMounts: []v1.VolumeMount{ + VolumeMounts: []corev1.VolumeMount{ { Name: "device-plugin", MountPath: "/var/lib/kubelet/device-plugins", @@ -471,11 +457,11 @@ func ExpectHabanaDevicePluginCreated() { }, }, }, - Volumes: []v1.Volume{ + Volumes: []corev1.Volume{ { Name: "device-plugin", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ Path: "/var/lib/kubelet/device-plugins", }, }, @@ -503,7 +489,7 @@ func ExpectEFADevicePluginCreated() { UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ Type: appsv1.RollingUpdateDaemonSetStrategyType, }, - Template: v1.PodTemplateSpec{ + Template: corev1.PodTemplateSpec{ ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{ Annotations: map[string]string{ "scheduler.alpha.kubernetes.io/critical-pod": "", @@ -512,35 +498,35 @@ func ExpectEFADevicePluginCreated() { "name": "aws-efa-k8s-device-plugin", }, }), - Spec: v1.PodSpec{ + Spec: corev1.PodSpec{ NodeSelector: map[string]string{ "aws.amazon.com/efa": "true", }, - Tolerations: []v1.Toleration{ + Tolerations: []corev1.Toleration{ { Key: "CriticalAddonsOnly", - Operator: v1.TolerationOpExists, + Operator: corev1.TolerationOpExists, }, { Key: "aws.amazon.com/efa", - Operator: v1.TolerationOpExists, - Effect: v1.TaintEffectNoSchedule, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, }, }, PriorityClassName: "system-node-critical", HostNetwork: true, - Containers: []v1.Container{ + Containers: []corev1.Container{ { Name: "aws-efea-k8s-device-plugin", Image: "602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/aws-efa-k8s-device-plugin:v0.3.3", - SecurityContext: &v1.SecurityContext{ + SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: lo.ToPtr(false), - Capabilities: &v1.Capabilities{ - Drop: []v1.Capability{"ALL"}, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, }, RunAsNonRoot: lo.ToPtr(false), }, - VolumeMounts: []v1.VolumeMount{ + VolumeMounts: []corev1.VolumeMount{ { Name: "device-plugin", MountPath: "/var/lib/kubelet/device-plugins", @@ -548,11 +534,11 @@ func ExpectEFADevicePluginCreated() { }, }, }, - Volumes: []v1.Volume{ + Volumes: []corev1.Volume{ { Name: "device-plugin", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ Path: "/var/lib/kubelet/device-plugins", }, }, diff --git a/test/suites/integration/hash_test.go b/test/suites/integration/hash_test.go index fc12376986be..7583546d1756 100644 --- a/test/suites/integration/hash_test.go +++ b/test/suites/integration/hash_test.go @@ -17,9 +17,9 @@ package integration_test import ( "sigs.k8s.io/controller-runtime/pkg/client" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -30,11 +30,11 @@ var _ = Describe("CRD Hash", func() { env.ExpectCreated(nodeClass, nodePool) Eventually(func(g Gomega) { - np := &corev1beta1.NodePool{} + np := &karpv1.NodePool{} err := env.Client.Get(env, client.ObjectKeyFromObject(nodePool), np) g.Expect(err).ToNot(HaveOccurred()) - hash, found := np.Annotations[corev1beta1.NodePoolHashAnnotationKey] + hash, found := np.Annotations[karpv1.NodePoolHashAnnotationKey] g.Expect(found).To(BeTrue()) g.Expect(hash).To(Equal(np.Hash())) }) @@ -43,11 +43,11 @@ var _ = Describe("CRD Hash", func() { env.ExpectCreated(nodeClass) Eventually(func(g Gomega) { - nc := &v1beta1.EC2NodeClass{} + nc := &v1.EC2NodeClass{} err := env.Client.Get(env, client.ObjectKeyFromObject(nodeClass), nc) g.Expect(err).ToNot(HaveOccurred()) - hash, found := nc.Annotations[v1beta1.AnnotationEC2NodeClassHash] + hash, found := nc.Annotations[v1.AnnotationEC2NodeClassHash] g.Expect(found).To(BeTrue()) g.Expect(hash).To(Equal(nc.Hash())) }) diff --git a/test/suites/integration/instance_profile_test.go b/test/suites/integration/instance_profile_test.go index 4c2e7b860a89..80bfc957018c 100644 --- a/test/suites/integration/instance_profile_test.go +++ b/test/suites/integration/instance_profile_test.go @@ -16,6 +16,12 @@ package integration_test import ( "fmt" + "time" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + + "github.com/awslabs/operatorpkg/status" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" @@ -25,6 +31,7 @@ import ( awserrors "github.com/aws/karpenter-provider-aws/pkg/errors" + . "github.com/awslabs/operatorpkg/test/expectations" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -81,5 +88,13 @@ var _ = Describe("InstanceProfile Generation", func() { instance := env.GetInstance(node.Name) Expect(instance.IamInstanceProfile).ToNot(BeNil()) Expect(lo.FromPtr(instance.IamInstanceProfile.Arn)).To(ContainSubstring(nodeClass.Status.InstanceProfile)) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: v1.ConditionTypeInstanceProfileReady, Status: metav1.ConditionTrue}) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: status.ConditionReady, Status: metav1.ConditionTrue}) + }) + It("should have the EC2NodeClass status as not ready since Instance Profile was not resolved", func() { + nodeClass.Spec.Role = fmt.Sprintf("KarpenterNodeRole-%s", "invalidRole") + env.ExpectCreated(nodeClass) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: v1.ConditionTypeInstanceProfileReady, Status: metav1.ConditionUnknown}) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: status.ConditionReady, Status: metav1.ConditionUnknown}) }) }) diff --git a/test/suites/integration/kubelet_config_test.go b/test/suites/integration/kubelet_config_test.go index a79804cdab0d..b7f06f3175c8 100644 --- a/test/suites/integration/kubelet_config_test.go +++ b/test/suites/integration/kubelet_config_test.go @@ -18,43 +18,40 @@ import ( "math" "time" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "knative.dev/pkg/ptr" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/aws/aws-sdk-go/service/ssm" "github.com/samber/lo" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" ) var _ = Describe("KubeletConfiguration Overrides", func() { Context("All kubelet configuration set", func() { BeforeEach(func() { // MaxPods needs to account for the daemonsets that will run on the nodes - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - MaxPods: ptr.Int32(110), - PodsPerCore: ptr.Int32(10), + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + MaxPods: lo.ToPtr(int32(110)), + PodsPerCore: lo.ToPtr(int32(10)), SystemReserved: map[string]string{ - string(v1.ResourceCPU): "200m", - string(v1.ResourceMemory): "200Mi", - string(v1.ResourceEphemeralStorage): "1Gi", + string(corev1.ResourceCPU): "200m", + string(corev1.ResourceMemory): "200Mi", + string(corev1.ResourceEphemeralStorage): "1Gi", }, KubeReserved: map[string]string{ - string(v1.ResourceCPU): "200m", - string(v1.ResourceMemory): "200Mi", - string(v1.ResourceEphemeralStorage): "1Gi", + string(corev1.ResourceCPU): "200m", + string(corev1.ResourceMemory): "200Mi", + string(corev1.ResourceEphemeralStorage): "1Gi", }, EvictionHard: map[string]string{ "memory.available": "5%", @@ -80,89 +77,75 @@ var _ = Describe("KubeletConfiguration Overrides", func() { "imagefs.inodesFree": {Duration: time.Minute * 2}, "pid.available": {Duration: time.Minute * 2}, }, - EvictionMaxPodGracePeriod: ptr.Int32(120), - ImageGCHighThresholdPercent: ptr.Int32(50), - ImageGCLowThresholdPercent: ptr.Int32(10), - CPUCFSQuota: ptr.Bool(false), + EvictionMaxPodGracePeriod: lo.ToPtr(int32(120)), + ImageGCHighThresholdPercent: lo.ToPtr(int32(50)), + ImageGCLowThresholdPercent: lo.ToPtr(int32(10)), + CPUCFSQuota: lo.ToPtr(false), } }) DescribeTable("Linux AMIFamilies", - func(amiFamily *string) { - nodeClass.Spec.AMIFamily = amiFamily + func(term v1.AMISelectorTerm) { + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{term} // TODO (jmdeal@): remove once 22.04 AMIs are supported - if *amiFamily == v1beta1.AMIFamilyUbuntu && env.GetK8sVersion(0) == "1.29" { - nodeClass.Spec.AMISelectorTerms = lo.Map([]string{ - "/aws/service/canonical/ubuntu/eks/20.04/1.28/stable/current/amd64/hvm/ebs-gp2/ami-id", - "/aws/service/canonical/ubuntu/eks/20.04/1.28/stable/current/arm64/hvm/ebs-gp2/ami-id", - }, func(arg string, _ int) v1beta1.AMISelectorTerm { - parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{Name: lo.ToPtr(arg)}) - Expect(err).To(BeNil()) - return v1beta1.AMISelectorTerm{ID: *parameter.Parameter.Value} - }) - } pod := test.Pod(test.PodOptions{ NodeSelector: map[string]string{ - v1.LabelOSStable: string(v1.Linux), - v1.LabelArchStable: "amd64", + corev1.LabelOSStable: string(corev1.Linux), + corev1.LabelArchStable: "amd64", }, }) env.ExpectCreated(nodeClass, nodePool, pod) env.EventuallyExpectHealthy(pod) env.ExpectCreatedNodeCount("==", 1) }, - Entry("when the AMIFamily is AL2", &v1beta1.AMIFamilyAL2), - Entry("when the AMIFamily is AL2023", &v1beta1.AMIFamilyAL2023), - Entry("when the AMIFamily is Ubuntu", &v1beta1.AMIFamilyUbuntu), - Entry("when the AMIFamily is Bottlerocket", &v1beta1.AMIFamilyBottlerocket), + Entry("when the AMIFamily is AL2", v1.AMISelectorTerm{Alias: "al2@latest"}), + Entry("when the AMIFamily is AL2023", v1.AMISelectorTerm{Alias: "al2023@latest"}), + Entry("when the AMIFamily is Bottlerocket", v1.AMISelectorTerm{Alias: "bottlerocket@latest"}), ) DescribeTable("Windows AMIFamilies", - func(amiFamily *string) { + func(term v1.AMISelectorTerm) { env.ExpectWindowsIPAMEnabled() DeferCleanup(func() { env.ExpectWindowsIPAMDisabled() }) - nodeClass.Spec.AMIFamily = amiFamily + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{term} // Need to enable nodepool-level OS-scoping for now since DS evaluation is done off of the nodepool // requirements, not off of the instance type options so scheduling can fail if nodepool aren't // properly scoped - // TODO: remove this requirement once VPC RC rolls out m7a.*, r7a.*, c7a.* ENI data (https://github.com/aws/karpenter-provider-aws/issues/4472) test.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, - Values: aws.ExcludedInstanceFamilies, - }, - }, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Windows)}, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Windows)}, }, }, ) pod := test.Pod(test.PodOptions{ Image: aws.WindowsDefaultImage, NodeSelector: map[string]string{ - v1.LabelOSStable: string(v1.Windows), - v1.LabelArchStable: "amd64", + corev1.LabelOSStable: string(corev1.Windows), + corev1.LabelArchStable: "amd64", }, }) env.ExpectCreated(nodeClass, nodePool, pod) env.EventuallyExpectHealthyWithTimeout(time.Minute*15, pod) env.ExpectCreatedNodeCount("==", 1) }, - Entry("when the AMIFamily is Windows2019", &v1beta1.AMIFamilyWindows2019), - Entry("when the AMIFamily is Windows2022", &v1beta1.AMIFamilyWindows2022), + // Windows tests are can flake due to the instance types that are used in testing. + // The VPC Resource controller will need to support the instance types that are used. + // If the instance type is not supported by the controller resource `vpc.amazonaws.com/PrivateIPv4Address` will not register. + // Issue: https://github.com/aws/karpenter-provider-aws/issues/4472 + // See: https://github.com/aws/amazon-vpc-resource-controller-k8s/blob/master/pkg/aws/vpc/limits.go + Entry("when the AMIFamily is Windows2019", v1.AMISelectorTerm{Alias: "windows2019@latest"}), + Entry("when the AMIFamily is Windows2022", v1.AMISelectorTerm{Alias: "windows2022@latest"}), ) }) It("should schedule pods onto separate nodes when maxPods is set", func() { // Get the DS pod count and use it to calculate the DS pod overhead dsCount := env.GetDaemonSetCount(nodePool) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - MaxPods: ptr.Int32(1 + int32(dsCount)), + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + MaxPods: lo.ToPtr(1 + int32(dsCount)), } numPods := 3 @@ -172,8 +155,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "large-app"}, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("100m")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, }, }, }) @@ -188,10 +171,10 @@ var _ = Describe("KubeletConfiguration Overrides", func() { // PodsPerCore needs to account for the daemonsets that will run on the nodes // This will have 4 pods available on each node (2 taken by daemonset pods) test.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCPU, - Operator: v1.NodeSelectorOpIn, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCPU, + Operator: corev1.NodeSelectorOpIn, Values: []string{"2"}, }, }, @@ -203,8 +186,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "large-app"}, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("100m")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, }, }, }) @@ -219,8 +202,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { // Since we restrict node to two cores, we will allow 6 pods. Both nodes will have // 4 DS pods and 2 test pods. dsCount := env.GetDaemonSetCount(nodePool) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - PodsPerCore: ptr.Int32(int32(math.Ceil(float64(2+dsCount) / 2))), + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + PodsPerCore: lo.ToPtr(int32(math.Ceil(float64(2+dsCount) / 2))), } env.ExpectCreated(nodeClass, nodePool, dep) @@ -229,20 +212,20 @@ var _ = Describe("KubeletConfiguration Overrides", func() { env.EventuallyExpectUniqueNodeNames(selector, 2) }) It("should ignore podsPerCore value when Bottlerocket is used", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} // All pods should schedule to a single node since we are ignoring podsPerCore value // This would normally schedule to 3 nodes if not using Bottlerocket test.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCPU, - Operator: v1.NodeSelectorOpIn, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCPU, + Operator: corev1.NodeSelectorOpIn, Values: []string{"2"}, }, }, ) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{PodsPerCore: ptr.Int32(1)} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{PodsPerCore: lo.ToPtr(int32(1))} numPods := 6 dep := test.Deployment(test.DeploymentOptions{ Replicas: int32(numPods), @@ -250,8 +233,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "large-app"}, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("100m")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, }, }, }) diff --git a/test/suites/integration/launch_template_test.go b/test/suites/integration/launch_template_test.go index 41f602105ad7..c9fd58dbc84d 100644 --- a/test/suites/integration/launch_template_test.go +++ b/test/suites/integration/launch_template_test.go @@ -21,7 +21,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" coretest "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -38,7 +38,7 @@ var _ = Describe("Launch Template Deletion", func() { Eventually(func(g Gomega) { output, _ := env.EC2API.DescribeLaunchTemplatesWithContext(env.Context, &ec2.DescribeLaunchTemplatesInput{ Filters: []*ec2.Filter{ - {Name: aws.String(fmt.Sprintf("tag:%s", v1beta1.LabelNodeClass)), Values: []*string{aws.String(nodeClass.Name)}}, + {Name: aws.String(fmt.Sprintf("tag:%s", v1.LabelNodeClass)), Values: []*string{aws.String(nodeClass.Name)}}, }, }) g.Expect(output.LaunchTemplates).To(HaveLen(0)) diff --git a/test/suites/integration/lease_garbagecollection_test.go b/test/suites/integration/lease_garbagecollection_test.go index d22774c2d55d..bcd1c42c0c05 100644 --- a/test/suites/integration/lease_garbagecollection_test.go +++ b/test/suites/integration/lease_garbagecollection_test.go @@ -18,7 +18,7 @@ import ( "time" coordinationsv1 "k8s.io/api/coordination/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/karpenter/pkg/test" @@ -29,8 +29,8 @@ var _ = Describe("Lease Garbage Collection", func() { var badLease *coordinationsv1.Lease BeforeEach(func() { badLease = &coordinationsv1.Lease{ - ObjectMeta: v1.ObjectMeta{ - CreationTimestamp: v1.Time{Time: time.Now().Add(-time.Hour * 2)}, + ObjectMeta: metav1.ObjectMeta{ + CreationTimestamp: metav1.Time{Time: time.Now().Add(-time.Hour * 2)}, Name: "new-lease", Namespace: "kube-node-lease", Labels: map[string]string{test.DiscoveryLabel: "unspecified"}, diff --git a/test/suites/integration/network_interface_test.go b/test/suites/integration/network_interface_test.go index 8a5976fed031..443913cc95d6 100644 --- a/test/suites/integration/network_interface_test.go +++ b/test/suites/integration/network_interface_test.go @@ -20,14 +20,14 @@ import ( "github.com/samber/lo" "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" ) var _ = Describe("NetworkInterfaces", func() { DescribeTable( "should correctly configure public IP assignment on instances", func(associatePublicIPAddress *bool) { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{{ Tags: map[string]string{ "Name": "*Private*", "karpenter.sh/discovery": env.ClusterName, diff --git a/test/suites/integration/scheduling_test.go b/test/suites/integration/scheduling_test.go deleted file mode 100644 index 9c9dba0f6a26..000000000000 --- a/test/suites/integration/scheduling_test.go +++ /dev/null @@ -1,558 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package integration_test - -import ( - "fmt" - "time" - - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/sets" - "knative.dev/pkg/ptr" - - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "sigs.k8s.io/karpenter/pkg/test" - - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" - "github.com/aws/karpenter-provider-aws/test/pkg/debug" - "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { - var selectors sets.Set[string] - - BeforeEach(func() { - // Make the NodePool requirements fully flexible, so we can match well-known label keys - nodePool = test.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpExists, - }, - }, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceGeneration, - Operator: v1.NodeSelectorOpExists, - }, - }, - ) - }) - BeforeAll(func() { - selectors = sets.New[string]() - }) - AfterAll(func() { - // Ensure that we're exercising all well known labels - Expect(lo.Keys(selectors)).To(ContainElements(append(corev1beta1.WellKnownLabels.UnsortedList(), lo.Keys(corev1beta1.NormalizedLabels)...))) - }) - - It("should apply annotations to the node", func() { - nodePool.Spec.Template.Annotations = map[string]string{ - "foo": "bar", - corev1beta1.DoNotDisruptAnnotationKey: "true", - } - pod := test.Pod() - env.ExpectCreated(nodeClass, nodePool, pod) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - Expect(env.GetNode(pod.Spec.NodeName).Annotations).To(And(HaveKeyWithValue("foo", "bar"), HaveKeyWithValue(corev1beta1.DoNotDisruptAnnotationKey, "true"))) - }) - - Context("Labels", func() { - It("should support well-known labels for instance type selection", func() { - nodeSelector := map[string]string{ - // Well Known - corev1beta1.NodePoolLabelKey: nodePool.Name, - v1.LabelInstanceTypeStable: "c5.large", - // Well Known to AWS - v1beta1.LabelInstanceHypervisor: "nitro", - v1beta1.LabelInstanceCategory: "c", - v1beta1.LabelInstanceGeneration: "5", - v1beta1.LabelInstanceFamily: "c5", - v1beta1.LabelInstanceSize: "large", - v1beta1.LabelInstanceCPU: "2", - v1beta1.LabelInstanceCPUManufacturer: "intel", - v1beta1.LabelInstanceMemory: "4096", - v1beta1.LabelInstanceNetworkBandwidth: "750", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for local NVME storage", func() { - selectors.Insert(v1beta1.LabelInstanceLocalNVME) // Add node selector keys to selectors used in testing to ensure we test all labels - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodePreferences: []v1.NodeSelectorRequirement{ - { - Key: v1beta1.LabelInstanceLocalNVME, - Operator: v1.NodeSelectorOpGt, - Values: []string{"0"}, - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1beta1.LabelInstanceLocalNVME, - Operator: v1.NodeSelectorOpGt, - Values: []string{"0"}, - }, - }, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for encryption in transit", func() { - selectors.Insert(v1beta1.LabelInstanceEncryptionInTransitSupported) // Add node selector keys to selectors used in testing to ensure we test all labels - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodePreferences: []v1.NodeSelectorRequirement{ - { - Key: v1beta1.LabelInstanceEncryptionInTransitSupported, - Operator: v1.NodeSelectorOpIn, - Values: []string{"true"}, - }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1beta1.LabelInstanceEncryptionInTransitSupported, - Operator: v1.NodeSelectorOpIn, - Values: []string{"true"}, - }, - }, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known deprecated labels", func() { - nodeSelector := map[string]string{ - // Deprecated Labels - v1.LabelFailureDomainBetaRegion: env.Region, - v1.LabelFailureDomainBetaZone: fmt.Sprintf("%sa", env.Region), - "topology.ebs.csi.aws.com/zone": fmt.Sprintf("%sa", env.Region), - - "beta.kubernetes.io/arch": "amd64", - "beta.kubernetes.io/os": "linux", - v1.LabelInstanceType: "c5.large", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for topology and architecture", func() { - nodeSelector := map[string]string{ - // Well Known - corev1beta1.NodePoolLabelKey: nodePool.Name, - v1.LabelTopologyRegion: env.Region, - v1.LabelTopologyZone: fmt.Sprintf("%sa", env.Region), - v1.LabelOSStable: "linux", - v1.LabelArchStable: "amd64", - corev1beta1.CapacityTypeLabelKey: corev1beta1.CapacityTypeOnDemand, - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for a gpu (nvidia)", func() { - nodeSelector := map[string]string{ - v1beta1.LabelInstanceGPUName: "t4", - v1beta1.LabelInstanceGPUMemory: "16384", - v1beta1.LabelInstanceGPUManufacturer: "nvidia", - v1beta1.LabelInstanceGPUCount: "1", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for an accelerator (inferentia)", func() { - nodeSelector := map[string]string{ - v1beta1.LabelInstanceAcceleratorName: "inferentia", - v1beta1.LabelInstanceAcceleratorManufacturer: "aws", - v1beta1.LabelInstanceAcceleratorCount: "1", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for windows-build version", func() { - env.ExpectWindowsIPAMEnabled() - DeferCleanup(func() { - env.ExpectWindowsIPAMDisabled() - }) - - nodeSelector := map[string]string{ - // Well Known - v1.LabelWindowsBuild: v1beta1.Windows2022Build, - v1.LabelOSStable: string(v1.Windows), // Specify the OS to enable vpc-resource-controller to inject the PrivateIPv4Address resource - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - Image: aws.WindowsDefaultImage, - }}) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 - // TODO: remove this requirement once VPC RC rolls out m7a.*, r7a.* ENI data (https://github.com/aws/karpenter-provider-aws/issues/4472) - test.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, - Values: aws.ExcludedInstanceFamilies, - }, - }, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Windows)}, - }, - }, - ) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCountWithTimeout(time.Minute*15, labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - DescribeTable("should support restricted label domain exceptions", func(domain string) { - // Assign labels to the nodepool so that it has known values - test.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: domain + "/team", Operator: v1.NodeSelectorOpExists}}, - corev1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: domain + "/custom-label", Operator: v1.NodeSelectorOpExists}}, - corev1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "subdomain." + domain + "/custom-label", Operator: v1.NodeSelectorOpExists}}, - ) - nodeSelector := map[string]string{ - domain + "/team": "team-1", - domain + "/custom-label": "custom-value", - "subdomain." + domain + "/custom-label": "custom-value", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - node := env.ExpectCreatedNodeCount("==", 1)[0] - // Ensure that the requirements/labels specified above are propagated onto the node - for k, v := range nodeSelector { - Expect(node.Labels).To(HaveKeyWithValue(k, v)) - } - }, - Entry("node-restriction.kuberentes.io", "node-restriction.kuberentes.io"), - Entry("node.kubernetes.io", "node.kubernetes.io"), - Entry("kops.k8s.io", "kops.k8s.io"), - ) - }) - - Context("Provisioning", func() { - It("should provision a node for naked pods", func() { - pod := test.Pod() - - env.ExpectCreated(nodeClass, nodePool, pod) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should provision a node for a deployment", Label(debug.NoWatch), Label(debug.NoEvents), func() { - deployment := test.Deployment(test.DeploymentOptions{Replicas: 50}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("<=", 2) // should probably all land on a single node, but at worst two depending on batching - }) - It("should provision a node for a self-affinity deployment", func() { - // just two pods as they all need to land on the same node - podLabels := map[string]string{"test": "self-affinity"} - deployment := test.Deployment(test.DeploymentOptions{ - Replicas: 2, - PodOptions: test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: podLabels, - }, - PodRequirements: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{MatchLabels: podLabels}, - TopologyKey: v1.LabelHostname, - }, - }, - }, - }) - - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), 2) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should provision three nodes for a zonal topology spread", func() { - // one pod per zone - podLabels := map[string]string{"test": "zonal-spread"} - deployment := test.Deployment(test.DeploymentOptions{ - Replicas: 3, - PodOptions: test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: podLabels, - }, - TopologySpreadConstraints: []v1.TopologySpreadConstraint{ - { - MaxSkew: 1, - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: podLabels}, - MinDomains: lo.ToPtr(int32(3)), - }, - }, - }, - }) - - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(podLabels), 3) - // Karpenter will launch three nodes, however if all three nodes don't get register with the cluster at the same time, two pods will be placed on one node. - // This can result in a case where all 3 pods are healthy, while there are only two created nodes. - // In that case, we still expect to eventually have three nodes. - env.EventuallyExpectNodeCount("==", 3) - }) - It("should provision a node using a NodePool with higher priority", func() { - nodePoolLowPri := test.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Weight: ptr.Int32(10), - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, - }, - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ - { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Linux)}, - }, - }, - { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"t3.nano"}, - }, - }, - }, - }, - }, - }, - }) - nodePoolHighPri := test.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Weight: ptr.Int32(100), - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, - }, - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ - { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Linux)}, - }, - }, - { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"c5.large"}, - }, - }, - }, - }, - }, - }, - }) - pod := test.Pod() - env.ExpectCreated(pod, nodeClass, nodePoolLowPri, nodePoolHighPri) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - Expect(ptr.StringValue(env.GetInstance(pod.Spec.NodeName).InstanceType)).To(Equal("c5.large")) - Expect(env.GetNode(pod.Spec.NodeName).Labels[corev1beta1.NodePoolLabelKey]).To(Equal(nodePoolHighPri.Name)) - }) - - DescribeTable( - "should provision a right-sized node when a pod has InitContainers (cpu)", - func(expectedNodeCPU string, containerRequirements v1.ResourceRequirements, initContainers ...v1.Container) { - if version, err := env.GetK8sMinorVersion(0); err != nil || version < 29 { - Skip("native sidecar containers are only enabled on EKS 1.29+") - } - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCPU, - Operator: v1.NodeSelectorOpIn, - Values: []string{"1", "2"}, - }, - }, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpNotIn, - Values: []string{"t"}, - }, - }) - pod := test.Pod(test.PodOptions{ - InitContainers: initContainers, - ResourceRequirements: containerRequirements, - }) - env.ExpectCreated(nodePool, nodeClass, pod) - env.EventuallyExpectHealthy(pod) - node := env.ExpectCreatedNodeCount("==", 1)[0] - Expect(node.ObjectMeta.GetLabels()[v1beta1.LabelInstanceCPU]).To(Equal(expectedNodeCPU)) - }, - Entry("sidecar requirements + later init requirements do exceed container requirements", "2", v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("400m")}, - }, ephemeralInitContainer(v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("300m")}, - }), v1.Container{ - RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("350m")}, - }, - }, ephemeralInitContainer(v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, - })), - Entry("sidecar requirements + later init requirements do not exceed container requirements", "1", v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("400m")}, - }, ephemeralInitContainer(v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("300m")}, - }), v1.Container{ - RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("350m")}, - }, - }, ephemeralInitContainer(v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("300m")}, - })), - Entry("init container requirements exceed all later requests", "2", v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("400m")}, - }, v1.Container{ - RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("100m")}, - }, - }, ephemeralInitContainer(v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1500m")}, - }), v1.Container{ - RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("100m")}, - }, - }), - ) - It("should provision a right-sized node when a pod has InitContainers (mixed resources)", func() { - if version, err := env.GetK8sMinorVersion(0); err != nil || version < 29 { - Skip("native sidecar containers are only enabled on EKS 1.29+") - } - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpNotIn, - Values: []string{"t"}, - }, - }) - pod := test.Pod(test.PodOptions{ - InitContainers: []v1.Container{ - { - RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), - Resources: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100m"), - v1.ResourceMemory: resource.MustParse("128Mi"), - }}, - }, - ephemeralInitContainer(v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("50m"), - v1.ResourceMemory: resource.MustParse("4Gi"), - }}), - }, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100m"), - v1.ResourceMemory: resource.MustParse("128Mi"), - }}, - }) - env.ExpectCreated(nodePool, nodeClass, pod) - env.EventuallyExpectHealthy(pod) - }) - }) -}) - -func ephemeralInitContainer(requirements v1.ResourceRequirements) v1.Container { - return v1.Container{ - Image: aws.EphemeralInitContainerImage, - Command: []string{"/bin/sh"}, - Args: []string{"-c", "sleep 5"}, - Resources: requirements, - } -} diff --git a/test/suites/integration/security_group_test.go b/test/suites/integration/security_group_test.go index 477fa1c70034..c4e0476495b1 100644 --- a/test/suites/integration/security_group_test.go +++ b/test/suites/integration/security_group_test.go @@ -18,15 +18,18 @@ import ( "time" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/awslabs/operatorpkg/status" "github.com/samber/lo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + . "github.com/awslabs/operatorpkg/test/expectations" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -35,8 +38,8 @@ var _ = Describe("SecurityGroups", func() { It("should use the security-group-id selector", func() { securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(securityGroups)).To(BeNumerically(">", 1)) - nodeClass.Spec.SecurityGroupSelectorTerms = lo.Map(securityGroups, func(sg aws.SecurityGroup, _ int) v1beta1.SecurityGroupSelectorTerm { - return v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = lo.Map(securityGroups, func(sg aws.SecurityGroup, _ int) v1.SecurityGroupSelectorTerm { + return v1.SecurityGroupSelectorTerm{ ID: lo.FromPtr(sg.GroupId), } }) @@ -55,7 +58,7 @@ var _ = Describe("SecurityGroups", func() { first := securityGroups[0] last := securityGroups[len(securityGroups)-1] - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"Name": lo.FromPtr(lo.FindOrElse(first.Tags, &ec2.Tag{}, func(tag *ec2.Tag) bool { return lo.FromPtr(tag.Key) == "Name" }).Value)}, }, @@ -75,10 +78,23 @@ var _ = Describe("SecurityGroups", func() { It("should update the EC2NodeClass status security groups", func() { env.ExpectCreated(nodeClass) EventuallyExpectSecurityGroups(env, nodeClass) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: v1.ConditionTypeSecurityGroupsReady, Status: metav1.ConditionTrue}) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: status.ConditionReady, Status: metav1.ConditionTrue}) + }) + + It("should have the NodeClass status as not ready since security groups were not resolved", func() { + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"karpenter.sh/discovery": "invalidName"}, + }, + } + env.ExpectCreated(nodeClass) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: v1.ConditionTypeSecurityGroupsReady, Status: metav1.ConditionFalse, Message: "SecurityGroupSelector did not match any SecurityGroups"}) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: status.ConditionReady, Status: metav1.ConditionFalse, Message: "SecurityGroupsReady=False"}) }) }) -func EventuallyExpectSecurityGroups(env *aws.Environment, nodeClass *v1beta1.EC2NodeClass) { +func EventuallyExpectSecurityGroups(env *aws.Environment, nodeClass *v1.EC2NodeClass) { securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(securityGroups).ToNot(HaveLen(0)) @@ -86,9 +102,9 @@ func EventuallyExpectSecurityGroups(env *aws.Environment, nodeClass *v1beta1.EC2 return lo.FromPtr(s.GroupId) })...) Eventually(func(g Gomega) { - temp := &v1beta1.EC2NodeClass{} + temp := &v1.EC2NodeClass{} g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClass), temp)).To(Succeed()) - g.Expect(sets.New(lo.Map(temp.Status.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) string { + g.Expect(sets.New(lo.Map(temp.Status.SecurityGroups, func(s v1.SecurityGroup, _ int) string { return s.ID })...).Equal(ids)) }).WithTimeout(10 * time.Second).Should(Succeed()) diff --git a/test/suites/integration/subnet_test.go b/test/suites/integration/subnet_test.go index 3312f91c33c6..d9a233390d9d 100644 --- a/test/suites/integration/subnet_test.go +++ b/test/suites/integration/subnet_test.go @@ -18,19 +18,22 @@ import ( "time" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/awslabs/operatorpkg/status" "github.com/onsi/gomega/types" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/karpenter/pkg/test" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + . "github.com/awslabs/operatorpkg/test/expectations" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -42,7 +45,7 @@ var _ = Describe("Subnets", func() { shuffledAZs := lo.Shuffle(lo.Keys(subnets)) firstSubnet := subnets[shuffledAZs[0]][0] - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { ID: firstSubnet, }, @@ -75,12 +78,12 @@ var _ = Describe("Subnets", func() { }) It("should use the subnet tag selector with multiple tag values", func() { // Get all the subnets for the cluster - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": env.ClusterName}) + subnets := env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(subnets)).To(BeNumerically(">", 1)) firstSubnet := subnets[0] lastSubnet := subnets[len(subnets)-1] - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { Tags: map[string]string{"Name": firstSubnet.Name}, }, @@ -102,9 +105,9 @@ var _ = Describe("Subnets", func() { Expect(len(subnets)).ToNot(Equal(0)) shuffledAZs := lo.Shuffle(lo.Keys(subnets)) - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelZoneFailureDomainStable, + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelZoneFailureDomainStable, Operator: "In", Values: []string{shuffledAZs[0]}, }}) @@ -122,6 +125,18 @@ var _ = Describe("Subnets", func() { It("should have the NodeClass status for subnets", func() { env.ExpectCreated(nodeClass) EventuallyExpectSubnets(env, nodeClass) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: v1.ConditionTypeSubnetsReady, Status: metav1.ConditionTrue}) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: status.ConditionReady, Status: metav1.ConditionTrue}) + }) + It("should have the NodeClass status as not ready since subnets were not resolved", func() { + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{"karpenter.sh/discovery": "invalidName"}, + }, + } + env.ExpectCreated(nodeClass) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: v1.ConditionTypeSubnetsReady, Status: metav1.ConditionFalse, Message: "SubnetSelector did not match any Subnets"}) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: status.ConditionReady, Status: metav1.ConditionFalse, Message: "SubnetsReady=False"}) }) }) @@ -171,15 +186,15 @@ type SubnetInfo struct { ID string } -func EventuallyExpectSubnets(env *aws.Environment, nodeClass *v1beta1.EC2NodeClass) { +func EventuallyExpectSubnets(env *aws.Environment, nodeClass *v1.EC2NodeClass) { subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(subnets).ToNot(HaveLen(0)) ids := sets.New(lo.Flatten(lo.Values(subnets))...) Eventually(func(g Gomega) { - temp := &v1beta1.EC2NodeClass{} + temp := &v1.EC2NodeClass{} g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClass), temp)).To(Succeed()) - g.Expect(sets.New(lo.Map(temp.Status.Subnets, func(s v1beta1.Subnet, _ int) string { + g.Expect(sets.New(lo.Map(temp.Status.Subnets, func(s v1.Subnet, _ int) string { return s.ID })...).Equal(ids)) }).WithTimeout(10 * time.Second).Should(Succeed()) diff --git a/test/suites/integration/suite_test.go b/test/suites/integration/suite_test.go index 9a2f70e8cd54..0330c3c929b9 100644 --- a/test/suites/integration/suite_test.go +++ b/test/suites/integration/suite_test.go @@ -17,9 +17,9 @@ package integration_test import ( "testing" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" . "github.com/onsi/ginkgo/v2" @@ -27,8 +27,8 @@ import ( ) var env *aws.Environment -var nodeClass *v1beta1.EC2NodeClass -var nodePool *corev1beta1.NodePool +var nodeClass *v1.EC2NodeClass +var nodePool *karpv1.NodePool func TestIntegration(t *testing.T) { RegisterFailHandler(Fail) diff --git a/test/suites/integration/tags_test.go b/test/suites/integration/tags_test.go index ab56b5abd312..df1049f46912 100644 --- a/test/suites/integration/tags_test.go +++ b/test/suites/integration/tags_test.go @@ -19,15 +19,17 @@ import ( "time" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/awslabs/operatorpkg/object" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coretest "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/test" @@ -54,23 +56,23 @@ var _ = Describe("Tags", func() { return ni.NetworkInterfaceId })...) - Expect(tagMap(instance.Tags)).To(HaveKeyWithValue("TestTag", "TestVal")) + Expect(instance.Tags).To(ContainElement(&ec2.Tag{Key: lo.ToPtr("TestTag"), Value: lo.ToPtr("TestVal")})) for _, volume := range volumes { - Expect(tagMap(volume.Tags)).To(HaveKeyWithValue("TestTag", "TestVal")) + Expect(volume.Tags).To(ContainElement(&ec2.Tag{Key: lo.ToPtr("TestTag"), Value: lo.ToPtr("TestVal")})) } for _, networkInterface := range networkInterfaces { // Any ENI that contains this createdAt tag was created by the VPC CNI DaemonSet - if _, ok := tagMap(networkInterface.TagSet)[createdAtTag]; !ok { - Expect(tagMap(networkInterface.TagSet)).To(HaveKeyWithValue("TestTag", "TestVal")) + if !lo.ContainsBy(networkInterface.TagSet, func(t *ec2.Tag) bool { return lo.FromPtr(t.Key) == createdAtTag }) { + Expect(networkInterface.TagSet).To(ContainElement(&ec2.Tag{Key: lo.ToPtr("TestTag"), Value: lo.ToPtr("TestVal")})) } } }) It("should tag spot instance requests when creating resources", func() { - coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, + coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeSpot}, }}) nodeClass.Spec.Tags = map[string]string{"TestTag": "TestVal"} pod := coretest.Pod() @@ -81,7 +83,41 @@ var _ = Describe("Tags", func() { instance := env.GetInstance(pod.Spec.NodeName) Expect(instance.SpotInstanceRequestId).ToNot(BeNil()) spotInstanceRequest := env.GetSpotInstanceRequest(instance.SpotInstanceRequestId) - Expect(tagMap(spotInstanceRequest.Tags)).To(HaveKeyWithValue("TestTag", "TestVal")) + Expect(spotInstanceRequest.Tags).To(ContainElement(&ec2.Tag{Key: lo.ToPtr("TestTag"), Value: lo.ToPtr("TestVal")})) + }) + It("should tag managed instance profiles", func() { + nodeClass.Spec.Tags["TestTag"] = "TestVal" + env.ExpectCreated(nodeClass) + + profile := env.EventuallyExpectInstanceProfileExists(env.GetInstanceProfileName(nodeClass)) + Expect(profile.Tags).To(ContainElements( + &iam.Tag{Key: lo.ToPtr(fmt.Sprintf("kubernetes.io/cluster/%s", env.ClusterName)), Value: lo.ToPtr("owned")}, + &iam.Tag{Key: lo.ToPtr(v1.LabelNodeClass), Value: lo.ToPtr(nodeClass.Name)}, + &iam.Tag{Key: lo.ToPtr(v1.EKSClusterNameTagKey), Value: lo.ToPtr(env.ClusterName)}, + )) + }) + It("should tag managed instance profiles with the eks:eks-cluster-name tag key after restart", func() { + env.ExpectCreated(nodeClass) + env.EventuallyExpectInstanceProfileExists(env.GetInstanceProfileName(nodeClass)) + + _, err := env.IAMAPI.UntagInstanceProfile(&iam.UntagInstanceProfileInput{ + InstanceProfileName: lo.ToPtr(env.GetInstanceProfileName(nodeClass)), + TagKeys: []*string{ + lo.ToPtr(v1.EKSClusterNameTagKey), + }, + }) + Expect(err).ToNot(HaveOccurred()) + + // Restart Karpenter to flush the instance profile cache and to trigger re-tagging of the instance profile + env.EventuallyExpectKarpenterRestarted() + + Eventually(func(g Gomega) { + out, err := env.IAMAPI.GetInstanceProfile(&iam.GetInstanceProfileInput{ + InstanceProfileName: lo.ToPtr(env.GetInstanceProfileName(nodeClass)), + }) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(out.InstanceProfile.Tags).To(ContainElement(&iam.Tag{Key: lo.ToPtr(v1.EKSClusterNameTagKey), Value: lo.ToPtr(env.ClusterName)})) + }).WithTimeout(time.Second * 20).Should(Succeed()) }) }) @@ -92,32 +128,36 @@ var _ = Describe("Tags", func() { env.ExpectCreated(nodePool, nodeClass, pod) env.EventuallyExpectCreatedNodeCount("==", 1) node := env.EventuallyExpectInitializedNodeCount("==", 1)[0] - nodeName := client.ObjectKeyFromObject(node) + nodeClaim := env.ExpectNodeClaimCount("==", 1)[0] Eventually(func(g Gomega) { - node = &v1.Node{} - g.Expect(env.Client.Get(env.Context, nodeName, node)).To(Succeed()) - g.Expect(node.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationInstanceTagged, "true")) - }, time.Minute) + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationInstanceTagged, "true")) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationClusterNameTaggedCompatability, "true")) + }, time.Minute).Should(Succeed()) nodeInstance := instance.NewInstance(lo.ToPtr(env.GetInstance(node.Name))) Expect(nodeInstance.Tags).To(HaveKeyWithValue("Name", node.Name)) - Expect(nodeInstance.Tags).To(HaveKey("karpenter.sh/nodeclaim")) + Expect(nodeInstance.Tags).To(HaveKeyWithValue("karpenter.sh/nodeclaim", nodeClaim.Name)) + Expect(nodeInstance.Tags).To(HaveKeyWithValue("eks:eks-cluster-name", env.ClusterName)) }) - It("shouldn't overwrite custom Name tags", func() { - nodeClass = test.EC2NodeClass(*nodeClass, v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{ + nodeClass = test.EC2NodeClass(*nodeClass, v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{ Tags: map[string]string{"Name": "custom-name", "testing/cluster": env.ClusterName}, }}) if env.PrivateCluster { nodeClass.Spec.Role = "" nodeClass.Spec.InstanceProfile = lo.ToPtr(fmt.Sprintf("KarpenterNodeInstanceProfile-%s", env.ClusterName)) } - nodePool = coretest.NodePool(*nodePool, corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{Name: nodeClass.Name}, + nodePool = coretest.NodePool(*nodePool, karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, }, }, }, @@ -127,23 +167,54 @@ var _ = Describe("Tags", func() { env.ExpectCreated(nodePool, nodeClass, pod) env.EventuallyExpectCreatedNodeCount("==", 1) node := env.EventuallyExpectInitializedNodeCount("==", 1)[0] - nodeName := client.ObjectKeyFromObject(node) + nodeClaim := env.ExpectNodeClaimCount("==", 1)[0] Eventually(func(g Gomega) { - node = &v1.Node{} - g.Expect(env.Client.Get(env.Context, nodeName, node)).To(Succeed()) - g.Expect(node.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationInstanceTagged, "true")) - }, time.Minute) + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationInstanceTagged, "true")) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationClusterNameTaggedCompatability, "true")) + }, time.Minute).Should(Succeed()) nodeInstance := instance.NewInstance(lo.ToPtr(env.GetInstance(node.Name))) Expect(nodeInstance.Tags).To(HaveKeyWithValue("Name", "custom-name")) - Expect(nodeInstance.Tags).To(HaveKey("karpenter.sh/nodeclaim")) + Expect(nodeInstance.Tags).To(HaveKeyWithValue("karpenter.sh/nodeclaim", nodeClaim.Name)) + Expect(nodeInstance.Tags).To(HaveKeyWithValue("eks:eks-cluster-name", env.ClusterName)) }) - }) -}) + It("should tag instance with eks:eks-cluster-name tag when the tag doesn't exist", func() { + pod := coretest.Pod() -func tagMap(tags []*ec2.Tag) map[string]string { - return lo.SliceToMap(tags, func(tag *ec2.Tag) (string, string) { - return *tag.Key, *tag.Value + env.ExpectCreated(nodePool, nodeClass, pod) + env.EventuallyExpectCreatedNodeCount("==", 1) + node := env.EventuallyExpectInitializedNodeCount("==", 1)[0] + nodeClaim := env.ExpectNodeClaimCount("==", 1)[0] + + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationInstanceTagged, "true")) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1.AnnotationClusterNameTaggedCompatability, "true")) + }, time.Minute).Should(Succeed()) + + _, err := env.EC2API.DeleteTags(&ec2.DeleteTagsInput{ + Resources: []*string{lo.ToPtr(env.ExpectParsedProviderID(node.Spec.ProviderID))}, + Tags: []*ec2.Tag{{Key: lo.ToPtr(v1.EKSClusterNameTagKey)}}, + }) + Expect(err).ToNot(HaveOccurred()) + + By(fmt.Sprintf("Removing the %s annotation to re-trigger tagging", v1.AnnotationClusterNameTaggedCompatability)) + Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + delete(nodeClaim.Annotations, v1.AnnotationClusterNameTaggedCompatability) + env.ExpectUpdated(nodeClaim) + + By(fmt.Sprintf("Polling for the %s tag update", v1.EKSClusterNameTagKey)) + Eventually(func(g Gomega) { + out, err := env.EC2API.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{lo.ToPtr(env.ExpectParsedProviderID(node.Spec.ProviderID))}, + }) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(out.Reservations).To(HaveLen(1)) + g.Expect(out.Reservations[0].Instances).To(HaveLen(1)) + g.Expect(out.Reservations[0].Instances[0].Tags).To(ContainElement(&ec2.Tag{Key: lo.ToPtr(v1.EKSClusterNameTagKey), Value: lo.ToPtr(env.ClusterName)})) + }).Should(Succeed()) + }) }) -} +}) diff --git a/test/suites/integration/utilization_test.go b/test/suites/integration/utilization_test.go index 7599de17d2c8..0d3c3154da03 100644 --- a/test/suites/integration/utilization_test.go +++ b/test/suites/integration/utilization_test.go @@ -17,15 +17,17 @@ package integration_test import ( "time" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/karpenter/pkg/test" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + "github.com/samber/lo" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/debug" . "github.com/onsi/ginkgo/v2" @@ -34,23 +36,35 @@ import ( var _ = Describe("Utilization", Label(debug.NoWatch), Label(debug.NoEvents), func() { It("should provision one pod per node", func() { test.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"t3.small"}, }, }, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpExists, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpExists, }, }, ) deployment := test.Deployment(test.DeploymentOptions{ - Replicas: 100, - PodOptions: test.PodOptions{ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1.5")}}}}) + Replicas: 100, + PodOptions: test.PodOptions{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: func() resource.Quantity { + dsOverhead := env.GetDaemonSetOverhead(nodePool) + base := lo.ToPtr(resource.MustParse("1800m")) + base.Sub(*dsOverhead.Cpu()) + return *base + }(), + }, + }, + }, + }) env.ExpectCreated(nodeClass, nodePool, deployment) env.EventuallyExpectHealthyPodCountWithTimeout(time.Minute*10, labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) diff --git a/test/suites/integration/validation_test.go b/test/suites/integration/validation_test.go index 790a880ce29b..76f4451408f0 100644 --- a/test/suites/integration/validation_test.go +++ b/test/suites/integration/validation_test.go @@ -16,16 +16,14 @@ package integration_test import ( "fmt" - "time" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - "knative.dev/pkg/ptr" + corev1 "k8s.io/api/core/v1" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coretest "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -35,7 +33,7 @@ var _ = Describe("Validation", func() { Context("NodePool", func() { It("should error when a restricted label is used in labels (karpenter.sh/nodepool)", func() { nodePool.Spec.Template.Labels = map[string]string{ - corev1beta1.NodePoolLabelKey: "my-custom-nodepool", + karpv1.NodePoolLabelKey: "my-custom-nodepool", } Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) @@ -47,93 +45,76 @@ var _ = Describe("Validation", func() { }) It("should allow a restricted label exception to be used in labels (node-restriction.kubernetes.io/custom-label)", func() { nodePool.Spec.Template.Labels = map[string]string{ - v1.LabelNamespaceNodeRestriction + "/custom-label": "custom-value", + corev1.LabelNamespaceNodeRestriction + "/custom-label": "custom-value", } Expect(env.Client.Create(env.Context, nodePool)).To(Succeed()) }) It("should allow a restricted label exception to be used in labels ([*].node-restriction.kubernetes.io/custom-label)", func() { nodePool.Spec.Template.Labels = map[string]string{ - "subdomain" + v1.LabelNamespaceNodeRestriction + "/custom-label": "custom-value", + "subdomain" + corev1.LabelNamespaceNodeRestriction + "/custom-label": "custom-value", } Expect(env.Client.Create(env.Context, nodePool)).To(Succeed()) }) It("should error when a requirement references a restricted label (karpenter.sh/nodepool)", func() { - nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.NodePoolLabelKey, - Operator: v1.NodeSelectorOpIn, + nodePool = coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.NodePoolLabelKey, + Operator: corev1.NodeSelectorOpIn, Values: []string{"default"}, }}) Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) It("should error when a requirement uses In but has no values", func() { - nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + nodePool = coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{}, }}) Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) It("should error when a requirement uses an unknown operator", func() { - nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, + nodePool = coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, Operator: "within", - Values: []string{corev1beta1.CapacityTypeSpot}, + Values: []string{karpv1.CapacityTypeSpot}, }}) Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) It("should error when Gt is used with multiple integer values", func() { - nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceMemory, - Operator: v1.NodeSelectorOpGt, + nodePool = coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceMemory, + Operator: corev1.NodeSelectorOpGt, Values: []string{"1000000", "2000000"}, }}) Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) It("should error when Lt is used with multiple integer values", func() { - nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceMemory, - Operator: v1.NodeSelectorOpLt, + nodePool = coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceMemory, + Operator: corev1.NodeSelectorOpLt, Values: []string{"1000000", "2000000"}, }}) Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) - It("should error when ttlSecondAfterEmpty is negative", func() { - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenEmpty - nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(-time.Second)} + It("should error when consolidateAfter is negative", func() { + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmpty + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("-1s") Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) - It("should error when ConsolidationPolicy=WhenUnderutilized is used with consolidateAfter", func() { - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenUnderutilized - nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Minute)} - Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) - }) - It("should error if imageGCHighThresholdPercent is less than imageGCLowThresholdPercent", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - ImageGCHighThresholdPercent: ptr.Int32(10), - ImageGCLowThresholdPercent: ptr.Int32(60), - } - Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) - }) - It("should error if imageGCHighThresholdPercent or imageGCLowThresholdPercent is negative", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - ImageGCHighThresholdPercent: ptr.Int32(-10), - } - Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - ImageGCLowThresholdPercent: ptr.Int32(-10), - } - Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) + It("should succeed when ConsolidationPolicy=WhenEmptyOrUnderutilized is used with consolidateAfter", func() { + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmptyOrUnderutilized + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("1m") + Expect(env.Client.Create(env.Context, nodePool)).To(Succeed()) }) It("should error when minValues for a requirement key is negative", func() { - nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + nodePool = coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"insance-type-1", "insance-type-2"}, }, MinValues: lo.ToPtr(-1)}, @@ -141,10 +122,10 @@ var _ = Describe("Validation", func() { Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) It("should error when minValues for a requirement key is zero", func() { - nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + nodePool = coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"insance-type-1", "insance-type-2"}, }, MinValues: lo.ToPtr(0)}, @@ -152,10 +133,10 @@ var _ = Describe("Validation", func() { Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) It("should error when minValues for a requirement key is more than 50", func() { - nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + nodePool = coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"insance-type-1", "insance-type-2"}, }, MinValues: lo.ToPtr(51)}, @@ -163,10 +144,10 @@ var _ = Describe("Validation", func() { Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) It("should error when minValues for a requirement key is greater than the values specified within In operator", func() { - nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + nodePool = coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"insance-type-1", "insance-type-2"}, }, MinValues: lo.ToPtr(3)}, @@ -175,12 +156,14 @@ var _ = Describe("Validation", func() { }) }) Context("EC2NodeClass", func() { - It("should error when amiSelectorTerms are not defined for amiFamily Custom", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom + It("should error when amiSelectorTerms are not defined", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{} Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) }) It("should fail for poorly formatted AMI ids", func() { - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ { ID: "must-start-with-ami", }, @@ -195,7 +178,7 @@ var _ = Describe("Validation", func() { nodeClass.Spec.Tags = map[string]string{"karpenter.sh/nodepool": "custom-value"} Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) - nodeClass.Spec.Tags = map[string]string{"karpenter.sh/managed-by": env.ClusterName} + nodeClass.Spec.Tags = map[string]string{v1.EKSClusterNameTagKey: env.ClusterName} Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) nodeClass.Spec.Tags = map[string]string{fmt.Sprintf("kubernetes.io/cluster/%s", env.ClusterName): "owned"} @@ -208,7 +191,7 @@ var _ = Describe("Validation", func() { Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) }) It("should fail when securityGroupSelectorTerms has id and other filters", func() { - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ { Tags: map[string]string{"karpenter.sh/discovery": env.ClusterName}, ID: "sg-12345", @@ -217,7 +200,7 @@ var _ = Describe("Validation", func() { Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) }) It("should fail when subnetSelectorTerms has id and other filters", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ { Tags: map[string]string{"karpenter.sh/discovery": env.ClusterName}, ID: "subnet-12345", @@ -226,7 +209,8 @@ var _ = Describe("Validation", func() { Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) }) It("should fail when amiSelectorTerms has id and other filters", func() { - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ { Tags: map[string]string{"karpenter.sh/discovery": env.ClusterName}, ID: "ami-12345", @@ -254,6 +238,12 @@ var _ = Describe("Validation", func() { Expect(env.Client.Update(env.Context, nodeClass)).ToNot(Succeed()) }) It("should fail to switch between a managed and unmanaged instance profile", func() { + // Skipping this test for private cluster because there is no VPC private endpoint for the IAM API. As a result, + // you cannot use the default spec.role field in your EC2NodeClass. Instead, you need to provision and manage an + // instance profile manually and then specify Karpenter to use this instance profile through the spec.instanceProfile field. + if env.PrivateCluster { + Skip("skipping Unmanaged instance profile test for private cluster") + } nodeClass.Spec.Role = "test-role" nodeClass.Spec.InstanceProfile = nil Expect(env.Client.Create(env.Context, nodeClass)).To(Succeed()) @@ -262,5 +252,22 @@ var _ = Describe("Validation", func() { nodeClass.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") Expect(env.Client.Update(env.Context, nodeClass)).ToNot(Succeed()) }) + It("should error if imageGCHighThresholdPercent is less than imageGCLowThresholdPercent", func() { + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + ImageGCHighThresholdPercent: lo.ToPtr(int32(10)), + ImageGCLowThresholdPercent: lo.ToPtr(int32(60)), + } + Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) + }) + It("should error if imageGCHighThresholdPercent or imageGCLowThresholdPercent is negative", func() { + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + ImageGCHighThresholdPercent: lo.ToPtr(int32(-10)), + } + Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + ImageGCLowThresholdPercent: lo.ToPtr(int32(-10)), + } + Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) + }) }) }) diff --git a/test/suites/interruption/suite_test.go b/test/suites/interruption/suite_test.go index f2adf66c1437..5c2d557d0ccc 100644 --- a/test/suites/interruption/suite_test.go +++ b/test/suites/interruption/suite_test.go @@ -20,16 +20,15 @@ import ( "time" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/uuid" - "knative.dev/pkg/ptr" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coretest "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/messages" "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/messages/scheduledchange" "github.com/aws/karpenter-provider-aws/pkg/operator/options" @@ -42,8 +41,8 @@ import ( ) var env *aws.Environment -var nodeClass *v1beta1.EC2NodeClass -var nodePool *corev1beta1.NodePool +var nodeClass *v1.EC2NodeClass +var nodePool *karpv1.NodePool func TestInterruption(t *testing.T) { RegisterFailHandler(Fail) @@ -70,11 +69,11 @@ var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Interruption", func() { It("should terminate the spot instance and spin-up a new node on spot interruption warning", func() { By("Creating a single healthy node with a healthy deployment") - nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, + nodePool = coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeSpot}, }}) numPods := 1 dep := coretest.Deployment(coretest.DeploymentOptions{ @@ -83,7 +82,7 @@ var _ = Describe("Interruption", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "my-app"}, }, - TerminationGracePeriodSeconds: ptr.Int64(0), + TerminationGracePeriodSeconds: lo.ToPtr(int64(0)), }, }) selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) @@ -117,7 +116,7 @@ var _ = Describe("Interruption", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "my-app"}, }, - TerminationGracePeriodSeconds: ptr.Int64(0), + TerminationGracePeriodSeconds: lo.ToPtr(int64(0)), }, }) selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) @@ -130,8 +129,8 @@ var _ = Describe("Interruption", func() { node := env.Monitor.CreatedNodes()[0] By("Stopping the EC2 instance without the EKS cluster's knowledge") - env.ExpectInstanceStopped(node.Name) // Make a call to the EC2 api to stop the instance - env.EventuallyExpectNotFoundAssertion(node).WithTimeout(time.Minute).Should(Succeed()) // shorten the timeout since we should react faster + env.ExpectInstanceStopped(node.Name) // Make a call to the EC2 api to stop the instance + env.EventuallyExpectNotFoundAssertion(node).Should(Succeed()) env.EventuallyExpectHealthyPodCount(selector, 1) }) It("should terminate the node at the API server when the EC2 instance is terminated", func() { @@ -143,7 +142,7 @@ var _ = Describe("Interruption", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "my-app"}, }, - TerminationGracePeriodSeconds: ptr.Int64(0), + TerminationGracePeriodSeconds: lo.ToPtr(int64(0)), }, }) selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) @@ -156,8 +155,8 @@ var _ = Describe("Interruption", func() { node := env.Monitor.CreatedNodes()[0] By("Terminating the EC2 instance without the EKS cluster's knowledge") - env.ExpectInstanceTerminated(node.Name) // Make a call to the EC2 api to stop the instance - env.EventuallyExpectNotFoundAssertion(node).WithTimeout(time.Minute).Should(Succeed()) // shorten the timeout since we should react faster + env.ExpectInstanceTerminated(node.Name) // Make a call to the EC2 api to stop the instance + env.EventuallyExpectNotFoundAssertion(node).Should(Succeed()) env.EventuallyExpectHealthyPodCount(selector, 1) }) It("should terminate the node when receiving a scheduled change health event", func() { @@ -169,7 +168,7 @@ var _ = Describe("Interruption", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "my-app"}, }, - TerminationGracePeriodSeconds: ptr.Int64(0), + TerminationGracePeriodSeconds: lo.ToPtr(int64(0)), }, }) selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) @@ -185,7 +184,7 @@ var _ = Describe("Interruption", func() { By("Creating a scheduled change health event in the SQS message queue") env.ExpectMessagesCreated(scheduledChangeMessage(env.Region, "000000000000", instanceID)) - env.EventuallyExpectNotFoundAssertion(node).WithTimeout(time.Minute).Should(Succeed()) // shorten the timeout since we should react faster + env.EventuallyExpectNotFoundAssertion(node).Should(Succeed()) env.EventuallyExpectHealthyPodCount(selector, 1) }) }) diff --git a/test/suites/ipv6/suite_test.go b/test/suites/ipv6/suite_test.go index f1b2f33ea104..980bfc8e1b73 100644 --- a/test/suites/ipv6/suite_test.go +++ b/test/suites/ipv6/suite_test.go @@ -19,12 +19,12 @@ import ( "testing" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coretest "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" . "github.com/onsi/ginkgo/v2" @@ -32,8 +32,8 @@ import ( ) var env *aws.Environment -var nodeClass *v1beta1.EC2NodeClass -var nodePool *corev1beta1.NodePool +var nodeClass *v1.EC2NodeClass +var nodePool *karpv1.NodePool func TestIPv6(t *testing.T) { RegisterFailHandler(Fail) @@ -51,16 +51,16 @@ var _ = BeforeEach(func() { nodeClass = env.DefaultEC2NodeClass() nodePool = env.DefaultNodePool(nodeClass) nodePool = coretest.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpExists, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpExists, }, }, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"t3a.small"}, }, }, @@ -76,21 +76,21 @@ var _ = Describe("IPv6", func() { env.EventuallyExpectHealthy(pod) env.ExpectCreatedNodeCount("==", 1) node := env.GetNode(pod.Spec.NodeName) - internalIPv6Addrs := lo.Filter(node.Status.Addresses, func(addr v1.NodeAddress, _ int) bool { - return addr.Type == v1.NodeInternalIP && net.ParseIP(addr.Address).To4() == nil + internalIPv6Addrs := lo.Filter(node.Status.Addresses, func(addr corev1.NodeAddress, _ int) bool { + return addr.Type == corev1.NodeInternalIP && net.ParseIP(addr.Address).To4() == nil }) Expect(internalIPv6Addrs).To(HaveLen(1)) }) It("should provision an IPv6 node by discovering kubeletConfig kube-dns IP", func() { clusterDNSAddr := env.ExpectIPv6ClusterDNS() - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ClusterDNS: []string{clusterDNSAddr}} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ClusterDNS: []string{clusterDNSAddr}} pod := coretest.Pod() env.ExpectCreated(pod, nodeClass, nodePool) env.EventuallyExpectHealthy(pod) env.ExpectCreatedNodeCount("==", 1) node := env.GetNode(pod.Spec.NodeName) - internalIPv6Addrs := lo.Filter(node.Status.Addresses, func(addr v1.NodeAddress, _ int) bool { - return addr.Type == v1.NodeInternalIP && net.ParseIP(addr.Address).To4() == nil + internalIPv6Addrs := lo.Filter(node.Status.Addresses, func(addr corev1.NodeAddress, _ int) bool { + return addr.Type == corev1.NodeInternalIP && net.ParseIP(addr.Address).To4() == nil }) Expect(internalIPv6Addrs).To(HaveLen(1)) }) diff --git a/test/suites/localzone/suite_test.go b/test/suites/localzone/suite_test.go index 53b0d4940773..e181465fe438 100644 --- a/test/suites/localzone/suite_test.go +++ b/test/suites/localzone/suite_test.go @@ -18,15 +18,15 @@ import ( "testing" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" . "github.com/onsi/ginkgo/v2" @@ -34,8 +34,8 @@ import ( ) var env *aws.Environment -var nodeClass *v1beta1.EC2NodeClass -var nodePool *corev1beta1.NodePool +var nodeClass *v1.EC2NodeClass +var nodePool *karpv1.NodePool func TestLocalZone(t *testing.T) { RegisterFailHandler(Fail) @@ -53,9 +53,9 @@ var _ = BeforeEach(func() { nodeClass = env.DefaultEC2NodeClass() // The majority of local zones do not support GP3. Feature support in local zones can be tracked here: // https://aws.amazon.com/about-aws/global-infrastructure/localzones/features/ - nodeClass.Spec.BlockDeviceMappings = append(nodeClass.Spec.BlockDeviceMappings, &v1beta1.BlockDeviceMapping{ + nodeClass.Spec.BlockDeviceMappings = append(nodeClass.Spec.BlockDeviceMappings, &v1.BlockDeviceMapping{ DeviceName: lo.ToPtr("/dev/xvda"), - EBS: &v1beta1.BlockDevice{ + EBS: &v1.BlockDevice{ VolumeSize: func() *resource.Quantity { quantity, err := resource.ParseQuantity("80Gi") Expect(err).To(BeNil()) @@ -66,12 +66,15 @@ var _ = BeforeEach(func() { }, }) nodePool = env.DefaultNodePool(nodeClass) - nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: lo.Keys(lo.PickByValues(env.GetZones(), []string{"local-zone"})), - }}) + nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelTopologyZone, + Operator: corev1.NodeSelectorOpIn, + Values: lo.FilterMap(env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName}), func(info aws.SubnetInfo, _ int) (string, bool) { + return info.Zone, info.ZoneType == "local-zone" + }), + }, + }) }) var _ = AfterEach(func() { env.Cleanup() }) var _ = AfterEach(func() { env.AfterEach() }) @@ -87,10 +90,10 @@ var _ = Describe("LocalZone", func() { ObjectMeta: metav1.ObjectMeta{ Labels: depLabels, }, - TopologySpreadConstraints: []v1.TopologySpreadConstraint{{ - TopologyKey: v1.LabelHostname, + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{ + TopologyKey: corev1.LabelHostname, MaxSkew: 1, - WhenUnsatisfiable: v1.DoNotSchedule, + WhenUnsatisfiable: corev1.DoNotSchedule, LabelSelector: &metav1.LabelSelector{ MatchLabels: depLabels, }, diff --git a/test/suites/nodeclaim/garbage_collection_test.go b/test/suites/nodeclaim/garbage_collection_test.go index 35d5dd4700bc..1dce2e540441 100644 --- a/test/suites/nodeclaim/garbage_collection_test.go +++ b/test/suites/nodeclaim/garbage_collection_test.go @@ -23,12 +23,12 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coretest "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" awserrors "github.com/aws/karpenter-provider-aws/pkg/errors" "github.com/aws/karpenter-provider-aws/pkg/utils" environmentaws "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" @@ -45,11 +45,11 @@ var _ = Describe("GarbageCollection", func() { BeforeEach(func() { securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": env.ClusterName}) + subnets := env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(securityGroups).ToNot(HaveLen(0)) Expect(subnets).ToNot(HaveLen(0)) - customAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) + customAMI = env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersion())) instanceProfileName = fmt.Sprintf("KarpenterNodeInstanceProfile-%s", env.ClusterName) roleName = fmt.Sprintf("KarpenterNodeRole-%s", env.ClusterName) instanceInput = &ec2.RunInstancesInput{ @@ -82,11 +82,11 @@ var _ = Describe("GarbageCollection", func() { Value: aws.String("owned"), }, { - Key: aws.String(corev1beta1.NodePoolLabelKey), + Key: aws.String(karpv1.NodePoolLabelKey), Value: aws.String(nodePool.Name), }, { - Key: aws.String(v1beta1.LabelNodeClass), + Key: aws.String(v1.LabelNodeClass), Value: aws.String(nodeClass.Name), }, }, @@ -98,10 +98,10 @@ var _ = Describe("GarbageCollection", func() { }) It("should succeed to garbage collect an Instance that was launched by a NodeClaim but has no Instance mapping", func() { // Update the userData for the instance input with the correct NodePool - rawContent, err := os.ReadFile("testdata/al2_userdata_input.sh") + rawContent, err := os.ReadFile("testdata/al2023_userdata_input.yaml") Expect(err).ToNot(HaveOccurred()) instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName, - env.ClusterEndpoint, env.ExpectCABundle(), nodePool.Name)))) + env.ClusterEndpoint, env.ExpectCABundle())))) env.ExpectInstanceProfileCreated(instanceProfileName, roleName) DeferCleanup(func() { @@ -125,12 +125,12 @@ var _ = Describe("GarbageCollection", func() { // Wait for the node to register with the cluster node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] - // Update the tags to add the karpenter.sh/managed-by tag + // Update the tags to add the EKSClusterNameTagKey tag _, err = env.EC2API.CreateTagsWithContext(env.Context, &ec2.CreateTagsInput{ Resources: []*string{out.Instances[0].InstanceId}, Tags: []*ec2.Tag{ { - Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Key: aws.String(v1.EKSClusterNameTagKey), Value: aws.String(env.ClusterName), }, }, @@ -140,12 +140,12 @@ var _ = Describe("GarbageCollection", func() { // Eventually expect the node and the instance to be removed (shutting-down) env.EventuallyExpectNotFound(node) Eventually(func(g Gomega) { - g.Expect(lo.FromPtr(env.GetInstanceByID(aws.StringValue(out.Instances[0].InstanceId)).State.Name)).To(Equal("shutting-down")) + g.Expect(lo.FromPtr(env.GetInstanceByID(aws.StringValue(out.Instances[0].InstanceId)).State.Name)).To(BeElementOf("terminated", "shutting-down")) }, time.Second*10).Should(Succeed()) }) It("should succeed to garbage collect an Instance that was deleted without the cluster's knowledge", func() { // Disable the interruption queue for the garbage collection coretest - env.ExpectSettingsOverridden(v1.EnvVar{Name: "INTERRUPTION_QUEUE", Value: ""}) + env.ExpectSettingsOverridden(corev1.EnvVar{Name: "INTERRUPTION_QUEUE", Value: ""}) pod := coretest.Pod() env.ExpectCreated(nodeClass, nodePool, pod) diff --git a/test/suites/nodeclaim/nodeclaim_test.go b/test/suites/nodeclaim/nodeclaim_test.go index bc36bd8fd380..ea4b54bbdb66 100644 --- a/test/suites/nodeclaim/nodeclaim_test.go +++ b/test/suites/nodeclaim/nodeclaim_test.go @@ -15,22 +15,22 @@ limitations under the License. package nodeclaim_test import ( - "encoding/base64" "fmt" "os" "time" - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/karpenter/pkg/utils/resources" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "github.com/awslabs/operatorpkg/object" + "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" - "sigs.k8s.io/karpenter/pkg/utils/resources" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -38,46 +38,50 @@ import ( var _ = Describe("StandaloneNodeClaim", func() { It("should create a standard NodeClaim within the 'c' instance family", func() { - nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodeClaim := test.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpIn, Values: []string{"c"}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, }, }, }, - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) env.ExpectCreated(nodeClass, nodeClaim) node := env.EventuallyExpectInitializedNodeCount("==", 1)[0] nodeClaim = env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] - Expect(node.Labels).To(HaveKeyWithValue(v1beta1.LabelInstanceCategory, "c")) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceCategory, "c")) env.EventuallyExpectNodeClaimsReady(nodeClaim) }) It("should create a standard NodeClaim based on resource requests", func() { - nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ - Spec: corev1beta1.NodeClaimSpec{ - Resources: corev1beta1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("3"), - v1.ResourceMemory: resource.MustParse("64Gi"), + nodeClaim := test.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + Resources: karpv1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), + corev1.ResourceMemory: resource.MustParse("64Gi"), }, }, - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) @@ -88,7 +92,7 @@ var _ = Describe("StandaloneNodeClaim", func() { env.EventuallyExpectNodeClaimsReady(nodeClaim) }) It("should create a NodeClaim propagating all the NodeClaim spec details", func() { - nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ + nodeClaim := test.NodeClaim(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "custom-annotation": "custom-value", @@ -97,68 +101,29 @@ var _ = Describe("StandaloneNodeClaim", func() { "custom-label": "custom-value", }, }, - Spec: corev1beta1.NodeClaimSpec{ - Taints: []v1.Taint{ + Spec: karpv1.NodeClaimSpec{ + Taints: []corev1.Taint{ { Key: "custom-taint", - Effect: v1.TaintEffectNoSchedule, + Effect: corev1.TaintEffectNoSchedule, Value: "custom-value", }, { Key: "other-custom-taint", - Effect: v1.TaintEffectNoExecute, + Effect: corev1.TaintEffectNoExecute, Value: "other-custom-value", }, }, - Resources: corev1beta1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("3"), - v1.ResourceMemory: resource.MustParse("16Gi"), - }, - }, - Kubelet: &corev1beta1.KubeletConfiguration{ - MaxPods: lo.ToPtr[int32](110), - PodsPerCore: lo.ToPtr[int32](10), - SystemReserved: map[string]string{ - string(v1.ResourceCPU): "200m", - string(v1.ResourceMemory): "200Mi", - string(v1.ResourceEphemeralStorage): "1Gi", - }, - KubeReserved: map[string]string{ - string(v1.ResourceCPU): "200m", - string(v1.ResourceMemory): "200Mi", - string(v1.ResourceEphemeralStorage): "1Gi", - }, - EvictionHard: map[string]string{ - "memory.available": "5%", - "nodefs.available": "5%", - "nodefs.inodesFree": "5%", - "imagefs.available": "5%", - "imagefs.inodesFree": "5%", - "pid.available": "3%", - }, - EvictionSoft: map[string]string{ - "memory.available": "10%", - "nodefs.available": "10%", - "nodefs.inodesFree": "10%", - "imagefs.available": "10%", - "imagefs.inodesFree": "10%", - "pid.available": "6%", + Resources: karpv1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), + corev1.ResourceMemory: resource.MustParse("16Gi"), }, - EvictionSoftGracePeriod: map[string]metav1.Duration{ - "memory.available": {Duration: time.Minute * 2}, - "nodefs.available": {Duration: time.Minute * 2}, - "nodefs.inodesFree": {Duration: time.Minute * 2}, - "imagefs.available": {Duration: time.Minute * 2}, - "imagefs.inodesFree": {Duration: time.Minute * 2}, - "pid.available": {Duration: time.Minute * 2}, - }, - EvictionMaxPodGracePeriod: lo.ToPtr[int32](120), - ImageGCHighThresholdPercent: lo.ToPtr[int32](50), - ImageGCLowThresholdPercent: lo.ToPtr[int32](10), }, - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) @@ -167,20 +132,20 @@ var _ = Describe("StandaloneNodeClaim", func() { Expect(node.Annotations).To(HaveKeyWithValue("custom-annotation", "custom-value")) Expect(node.Labels).To(HaveKeyWithValue("custom-label", "custom-value")) Expect(node.Spec.Taints).To(ContainElements( - v1.Taint{ + corev1.Taint{ Key: "custom-taint", - Effect: v1.TaintEffectNoSchedule, + Effect: corev1.TaintEffectNoSchedule, Value: "custom-value", }, - v1.Taint{ + corev1.Taint{ Key: "other-custom-taint", - Effect: v1.TaintEffectNoExecute, + Effect: corev1.TaintEffectNoExecute, Value: "other-custom-value", }, )) Expect(node.OwnerReferences).To(ContainElement( metav1.OwnerReference{ - APIVersion: corev1beta1.SchemeGroupVersion.String(), + APIVersion: object.GVK(nodeClaim).GroupVersion().String(), Kind: "NodeClaim", Name: nodeClaim.Name, UID: nodeClaim.UID, @@ -191,26 +156,28 @@ var _ = Describe("StandaloneNodeClaim", func() { env.EventuallyExpectNodeClaimsReady(nodeClaim) }) It("should remove the cloudProvider NodeClaim when the cluster NodeClaim is deleted", func() { - nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodeClaim := test.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpIn, Values: []string{"c"}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, }, }, }, - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) @@ -226,30 +193,32 @@ var _ = Describe("StandaloneNodeClaim", func() { env.EventuallyExpectNotFound(nodeClaim, node) Eventually(func(g Gomega) { - g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(Equal("shutting-down")) + g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(BeElementOf("terminated", "shutting-down")) }, time.Second*10).Should(Succeed()) }) It("should delete a NodeClaim from the node termination finalizer", func() { - nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodeClaim := test.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpIn, Values: []string{"c"}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, }, }, }, - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) @@ -265,48 +234,50 @@ var _ = Describe("StandaloneNodeClaim", func() { env.EventuallyExpectNotFound(nodeClaim, node) Eventually(func(g Gomega) { - g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(Equal("shutting-down")) + g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(BeElementOf("terminated", "shutting-down")) }, time.Second*10).Should(Succeed()) }) It("should create a NodeClaim with custom labels passed through the userData", func() { - customAMI := env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) + customAMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersion())) // Update the userData for the instance input with the correct NodePool - rawContent, err := os.ReadFile("testdata/al2_userdata_custom_labels_input.sh") + rawContent, err := os.ReadFile("testdata/al2023_userdata_custom_labels_input.yaml") Expect(err).ToNot(HaveOccurred()) - // Create userData that adds custom labels through the --kubelet-extra-args - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: customAMI}} - nodeClass.Spec.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName, - env.ClusterEndpoint, env.ExpectCABundle())))) + // Create userData that adds custom labels through the --node-labels + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: customAMI}} + nodeClass.Spec.UserData = lo.ToPtr(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle())) - nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodeClaim := test.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpIn, Values: []string{"c"}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"amd64"}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, }, }, }, - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) @@ -319,45 +290,47 @@ var _ = Describe("StandaloneNodeClaim", func() { env.EventuallyExpectNodeClaimsReady(nodeClaim) }) It("should delete a NodeClaim after the registration timeout when the node doesn't register", func() { - customAMI := env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) + customAMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersion())) // Update the userData for the instance input with the correct NodePool - rawContent, err := os.ReadFile("testdata/al2_userdata_input.sh") + rawContent, err := os.ReadFile("testdata/al2023_userdata_input.yaml") Expect(err).ToNot(HaveOccurred()) - // Create userData that adds custom labels through the --kubelet-extra-args - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: customAMI}} + // Create userData that adds custom labels through the --node-labels + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: customAMI}} // Giving bad clusterName and clusterEndpoint to the userData - nodeClass.Spec.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), "badName", "badEndpoint", env.ExpectCABundle())))) + nodeClass.Spec.UserData = lo.ToPtr(fmt.Sprintf(string(rawContent), "badName", "badEndpoint", env.ExpectCABundle())) - nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodeClaim := test.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpIn, Values: []string{"c"}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"amd64"}, }, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, }, }, }, - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) @@ -367,14 +340,76 @@ var _ = Describe("StandaloneNodeClaim", func() { // Expect that the nodeClaim eventually launches and has false Registration/Initialization Eventually(func(g Gomega) { - temp := &corev1beta1.NodeClaim{} + temp := &karpv1.NodeClaim{} g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClaim), temp)).To(Succeed()) - g.Expect(temp.StatusConditions().GetCondition(corev1beta1.Launched).IsTrue()).To(BeTrue()) - g.Expect(temp.StatusConditions().GetCondition(corev1beta1.Registered).IsFalse()).To(BeTrue()) - g.Expect(temp.StatusConditions().GetCondition(corev1beta1.Initialized).IsFalse()).To(BeTrue()) + g.Expect(temp.StatusConditions().Get(karpv1.ConditionTypeLaunched).IsTrue()).To(BeTrue()) + g.Expect(temp.StatusConditions().Get(karpv1.ConditionTypeRegistered).IsUnknown()).To(BeTrue()) + g.Expect(temp.StatusConditions().Get(karpv1.ConditionTypeInitialized).IsUnknown()).To(BeTrue()) }).Should(Succeed()) // Expect that the nodeClaim is eventually de-provisioned due to the registration timeout env.EventuallyExpectNotFoundAssertion(nodeClaim).WithTimeout(time.Minute * 20).Should(Succeed()) }) + It("should delete a NodeClaim if it references a NodeClass that doesn't exist", func() { + nodeClaim := test.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"c"}, + }, + }, + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, + }, + }, + }, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + }, + }) + // Don't create the NodeClass and expect that the NodeClaim fails and gets deleted + env.ExpectCreated(nodeClaim) + env.EventuallyExpectNotFound(nodeClaim) + }) + It("should delete a NodeClaim if it references a NodeClass that isn't Ready", func() { + nodeClaim := test.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"c"}, + }, + }, + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, + }, + }, + }, + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + }, + }) + // Point to an AMI that doesn't exist so that the NodeClass goes NotReady + nodeClass.Spec.AMIFamily = &v1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: "ami-123456789"}} + env.ExpectCreated(nodeClass, nodeClaim) + env.EventuallyExpectNotFound(nodeClaim) + }) }) diff --git a/test/suites/nodeclaim/suite_test.go b/test/suites/nodeclaim/suite_test.go index d5b49bcbed83..530b8faae9a1 100644 --- a/test/suites/nodeclaim/suite_test.go +++ b/test/suites/nodeclaim/suite_test.go @@ -17,9 +17,9 @@ package nodeclaim_test import ( "testing" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" . "github.com/onsi/ginkgo/v2" @@ -27,8 +27,8 @@ import ( ) var env *aws.Environment -var nodeClass *v1beta1.EC2NodeClass -var nodePool *corev1beta1.NodePool +var nodeClass *v1.EC2NodeClass +var nodePool *karpv1.NodePool func TestNodeClaim(t *testing.T) { RegisterFailHandler(Fail) diff --git a/test/suites/nodeclaim/testdata/al2023_userdata_custom_labels_input.yaml b/test/suites/nodeclaim/testdata/al2023_userdata_custom_labels_input.yaml new file mode 100644 index 000000000000..41034abd0bcf --- /dev/null +++ b/test/suites/nodeclaim/testdata/al2023_userdata_custom_labels_input.yaml @@ -0,0 +1,15 @@ +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: %s + apiServerEndpoint: %s + certificateAuthority: %s + cidr: 10.100.0.0/16 + kubelet: + config: + clusterDNS: + - 10.0.100.10 + flags: + - --node-labels='testing/cluster=unspecified,custom-label=custom-value,custom-label2=custom-value2' + - --register-with-taints='karpenter.sh/unregistered:NoExecute' \ No newline at end of file diff --git a/test/suites/nodeclaim/testdata/al2023_userdata_input.yaml b/test/suites/nodeclaim/testdata/al2023_userdata_input.yaml new file mode 100644 index 000000000000..e400f3259490 --- /dev/null +++ b/test/suites/nodeclaim/testdata/al2023_userdata_input.yaml @@ -0,0 +1,15 @@ +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: %s + apiServerEndpoint: %s + certificateAuthority: %s + cidr: 10.100.0.0/16 + kubelet: + config: + clusterDNS: + - 10.0.100.10 + flags: + - --node-labels='testing/cluster=unspecified' + - --register-with-taints='karpenter.sh/unregistered:NoExecute' diff --git a/test/suites/nodeclaim/testdata/al2_userdata_custom_labels_input.sh b/test/suites/nodeclaim/testdata/al2_userdata_custom_labels_input.sh deleted file mode 100644 index 86feeb4aa5d6..000000000000 --- a/test/suites/nodeclaim/testdata/al2_userdata_custom_labels_input.sh +++ /dev/null @@ -1,13 +0,0 @@ -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/x-shellscript; charset="us-ascii" - -#!/bin/bash -exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 -/etc/eks/bootstrap.sh '%s' --apiserver-endpoint '%s' --b64-cluster-ca '%s' \ ---use-max-pods false \ ---kubelet-extra-args '--node-labels=testing/cluster=unspecified,custom-label=custom-value,custom-label2=custom-value2' - ---BOUNDARY-- diff --git a/test/suites/nodeclaim/testdata/al2_userdata_input.sh b/test/suites/nodeclaim/testdata/al2_userdata_input.sh deleted file mode 100644 index 1fd3e27e30f0..000000000000 --- a/test/suites/nodeclaim/testdata/al2_userdata_input.sh +++ /dev/null @@ -1,13 +0,0 @@ -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/x-shellscript; charset="us-ascii" - -#!/bin/bash -exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 -/etc/eks/bootstrap.sh '%s' --apiserver-endpoint '%s' --b64-cluster-ca '%s' \ ---use-max-pods false \ ---kubelet-extra-args '--node-labels=karpenter.sh/nodepool=%s,testing/cluster=unspecified' - ---BOUNDARY-- diff --git a/test/suites/scale/deprovisioning_test.go b/test/suites/scale/deprovisioning_test.go index cb7086bbee08..96e7d67f5007 100644 --- a/test/suites/scale/deprovisioning_test.go +++ b/test/suites/scale/deprovisioning_test.go @@ -21,19 +21,19 @@ import ( "sync" "time" + "github.com/awslabs/operatorpkg/object" "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/uuid" - "knative.dev/pkg/ptr" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/messages" "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/messages/scheduledchange" awstest "github.com/aws/karpenter-provider-aws/pkg/test" @@ -67,35 +67,49 @@ const ( // disableProvisioningLimits represents limits that can be applied to a nodePool if you want a nodePool // that can deprovision nodes but cannot provision nodes -var disableProvisioningLimits = corev1beta1.Limits{ - v1.ResourceCPU: resource.MustParse("0"), - v1.ResourceMemory: resource.MustParse("0Gi"), +var disableProvisioningLimits = karpv1.Limits{ + corev1.ResourceCPU: resource.MustParse("0"), + corev1.ResourceMemory: resource.MustParse("0Gi"), } var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), func() { - var nodePool *corev1beta1.NodePool - var nodeClass *v1beta1.EC2NodeClass + var nodePool *karpv1.NodePool + var nodeClass *v1.EC2NodeClass var deployment *appsv1.Deployment var deploymentOptions test.DeploymentOptions var selector labels.Selector var dsCount int BeforeEach(func() { - env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=True"}) + env.ExpectSettingsOverridden(corev1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=True"}) nodeClass = env.DefaultEC2NodeClass() nodePool = env.DefaultNodePool(nodeClass) nodePool.Spec.Limits = nil - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1beta1.LabelInstanceHypervisor, - Operator: v1.NodeSelectorOpIn, - Values: []string{"nitro"}, - }}) + test.ReplaceRequirements(nodePool, []karpv1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: v1.LabelInstanceHypervisor, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"nitro"}, + }, + }, + // Ensure that all pods can fit on to the provisioned nodes including all daemonsets + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceSize, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"large"}, + }, + }, + }...) + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{{ + Nodes: "70%", + }} deploymentOptions = test.DeploymentOptions{ PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("50Mi"), + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("50Mi"), }, }, TerminationGracePeriodSeconds: lo.ToPtr[int64](0), @@ -134,12 +148,12 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), deploymentOptions.PodOptions.Labels = map[string]string{ deprovisioningTypeKey: v, } - deploymentOptions.PodOptions.Tolerations = []v1.Toleration{ + deploymentOptions.PodOptions.Tolerations = []corev1.Toleration{ { Key: deprovisioningTypeKey, - Operator: v1.TolerationOpEqual, + Operator: corev1.TolerationOpEqual, Value: v, - Effect: v1.TaintEffectNoSchedule, + Effect: corev1.TaintEffectNoSchedule, }, } deploymentOptions.Replicas = int32(replicas) @@ -147,24 +161,21 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), deploymentMap[v] = d } - nodePoolMap := map[string]*corev1beta1.NodePool{} + nodePoolMap := map[string]*karpv1.NodePool{} // Generate all the nodePools for multi-deprovisioning for _, v := range disruptionMethods { np := test.NodePool() np.Spec = *nodePool.Spec.DeepCopy() - np.Spec.Template.Spec.Taints = []v1.Taint{ + np.Spec.Template.Spec.Taints = []corev1.Taint{ { Key: deprovisioningTypeKey, Value: v, - Effect: v1.TaintEffectNoSchedule, + Effect: corev1.TaintEffectNoSchedule, }, } np.Spec.Template.Labels = map[string]string{ deprovisioningTypeKey: v, } - np.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), - } nodePoolMap[v] = test.NodePool(*np) } @@ -182,12 +193,17 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), } wg.Wait() + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), + } // Create a separate nodeClass for drift so that we can change the nodeClass later without it affecting // the other nodePools driftNodeClass := awstest.EC2NodeClass() driftNodeClass.Spec = *nodeClass.Spec.DeepCopy() - nodePoolMap[driftValue].Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{ - Name: driftNodeClass.Name, + nodePoolMap[driftValue].Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{ + Group: object.GVK(driftNodeClass).Group, + Kind: object.GVK(driftNodeClass).Kind, + Name: driftNodeClass.Name, } env.MeasureProvisioningDurationFor(func() { By("kicking off provisioning by applying the nodePool and nodeClass") @@ -234,13 +250,13 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), nodePoolMap[noExpirationValue].Spec = *nodePoolMap[expirationValue].Spec.DeepCopy() // Enable consolidation, emptiness, and expiration - nodePoolMap[consolidationValue].Spec.Disruption.ConsolidateAfter = nil - nodePoolMap[emptinessValue].Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenEmpty - nodePoolMap[emptinessValue].Spec.Disruption.ConsolidateAfter.Duration = ptr.Duration(0) - nodePoolMap[expirationValue].Spec.Disruption.ExpireAfter.Duration = ptr.Duration(0) + nodePoolMap[consolidationValue].Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") + nodePoolMap[emptinessValue].Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmpty + nodePoolMap[emptinessValue].Spec.Disruption.ConsolidateAfter.Duration = lo.ToPtr(time.Duration(0)) + nodePoolMap[expirationValue].Spec.Template.Spec.ExpireAfter.Duration = lo.ToPtr(time.Duration(0)) nodePoolMap[expirationValue].Spec.Limits = disableProvisioningLimits // Update the drift NodeClass to start drift on Nodes assigned to this NodeClass - driftNodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + driftNodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} // Create test assertions to ensure during the multiple deprovisioner scale-downs type testAssertions struct { @@ -265,7 +281,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), deletedCount: nodeCountPerNodePool, nodeCount: nodeCountPerNodePool, nodeCountSelector: labels.SelectorFromSet(map[string]string{ - corev1beta1.NodePoolLabelKey: nodePoolMap[noExpirationValue].Name, + karpv1.NodePoolLabelKey: nodePoolMap[noExpirationValue].Name, }), createdCount: nodeCountPerNodePool, }, @@ -284,6 +300,9 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), env.MeasureDeprovisioningDurationFor(func() { By("enabling deprovisioning across nodePools") for _, p := range nodePoolMap { + p.Spec.Disruption.Budgets = []karpv1.Budget{{ + Nodes: "70%", + }} env.ExpectCreatedOrUpdated(p) } env.ExpectUpdated(driftNodeClass) @@ -299,14 +318,14 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), // Provide a default selector based on the original nodePool name if one isn't specified selector = assertions.deletedNodeCountSelector if selector == nil { - selector = labels.SelectorFromSet(map[string]string{corev1beta1.NodePoolLabelKey: nodePoolMap[d].Name}) + selector = labels.SelectorFromSet(map[string]string{karpv1.NodePoolLabelKey: nodePoolMap[d].Name}) } env.EventuallyExpectDeletedNodeCountWithSelector("==", assertions.deletedCount, selector) // Provide a default selector based on the original nodePool name if one isn't specified selector = assertions.nodeCountSelector if selector == nil { - selector = labels.SelectorFromSet(map[string]string{corev1beta1.NodePoolLabelKey: nodePoolMap[d].Name}) + selector = labels.SelectorFromSet(map[string]string{karpv1.NodePoolLabelKey: nodePoolMap[d].Name}) } env.EventuallyExpectNodeCountWithSelector("==", assertions.nodeCount, selector) env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deploymentMap[d].Spec.Selector.MatchLabels), int(lo.FromPtr(deploymentMap[d].Spec.Replicas))) @@ -331,7 +350,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), replicas := replicasPerNode * expectedNodeCount deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), } @@ -363,8 +382,8 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), env.MeasureDeprovisioningDurationFor(func() { By("kicking off deprovisioning by setting the consolidation enabled value on the nodePool") - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenUnderutilized - nodePool.Spec.Disruption.ConsolidateAfter = nil + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmptyOrUnderutilized + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") env.ExpectUpdated(nodePool) env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) @@ -384,7 +403,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), replicas := replicasPerNode * expectedNodeCount deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), } @@ -417,8 +436,8 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), env.MeasureDeprovisioningDurationFor(func() { By("kicking off deprovisioning by setting the consolidation enabled value on the nodePool") - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenUnderutilized - nodePool.Spec.Disruption.ConsolidateAfter = nil + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmptyOrUnderutilized + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") env.ExpectUpdated(nodePool) env.EventuallyExpectDeletedNodeCount("==", int(float64(expectedNodeCount)*0.8)) @@ -438,21 +457,21 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), replicas := replicasPerNode * expectedNodeCount // Add in a instance type size requirement that's larger than the smallest that fits the pods. - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceSize, + Operator: corev1.NodeSelectorOpIn, Values: []string{"2xlarge"}, }}) deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) // Hostname anti-affinity to require one pod on each node - deployment.Spec.Template.Spec.Affinity = &v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + deployment.Spec.Template.Spec.Affinity = &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ { LabelSelector: deployment.Spec.Selector, - TopologyKey: v1.LabelHostname, + TopologyKey: corev1.LabelHostname, }, }, }, @@ -484,10 +503,10 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), By("kicking off deprovisioning by setting the consolidation enabled value on the nodePool") // The nodePool defaults to a larger instance type than we need so enabling consolidation and making // the requirements wide-open should cause deletes and increase our utilization on the cluster - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenUnderutilized - nodePool.Spec.Disruption.ConsolidateAfter = nil - nodePool.Spec.Template.Spec.Requirements = lo.Reject(nodePool.Spec.Template.Spec.Requirements, func(r corev1beta1.NodeSelectorRequirementWithMinValues, _ int) bool { - return r.Key == v1beta1.LabelInstanceSize + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmptyOrUnderutilized + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") + nodePool.Spec.Template.Spec.Requirements = lo.Reject(nodePool.Spec.Template.Spec.Requirements, func(r karpv1.NodeSelectorRequirementWithMinValues, _ int) bool { + return r.Key == v1.LabelInstanceSize }) env.ExpectUpdated(nodePool) @@ -511,7 +530,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), replicas := replicasPerNode * expectedNodeCount deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), } @@ -544,8 +563,8 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), env.MeasureDeprovisioningDurationFor(func() { By("kicking off deprovisioning emptiness by setting the ttlSecondsAfterEmpty value on the nodePool") - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenEmpty - nodePool.Spec.Disruption.ConsolidateAfter.Duration = ptr.Duration(0) + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmpty + nodePool.Spec.Disruption.ConsolidateAfter.Duration = lo.ToPtr(time.Duration(0)) env.ExpectCreatedOrUpdated(nodePool) env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) @@ -567,9 +586,11 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), replicas := replicasPerNode * expectedNodeCount deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), } + // Enable Expiration + nodePool.Spec.Template.Spec.ExpireAfter = karpv1.MustParseNillableDuration("5m") By("waiting for the deployment to deploy all of its pods") env.ExpectCreated(deployment) @@ -597,19 +618,14 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), By("kicking off deprovisioning expiration by setting the ttlSecondsUntilExpired value on the nodePool") // Change limits so that replacement nodes will use another nodePool. nodePool.Spec.Limits = disableProvisioningLimits - // Enable Expiration - nodePool.Spec.Disruption.ExpireAfter.Duration = ptr.Duration(0) noExpireNodePool := test.NodePool(*nodePool.DeepCopy()) // Disable Expiration - noExpireNodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{} - noExpireNodePool.Spec.Disruption.ExpireAfter.Duration = nil + noExpireNodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("Never") + noExpireNodePool.Spec.Template.Spec.ExpireAfter.Duration = nil noExpireNodePool.ObjectMeta = metav1.ObjectMeta{Name: test.RandomName()} - noExpireNodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), - } noExpireNodePool.Spec.Limits = nil env.ExpectCreatedOrUpdated(nodePool, noExpireNodePool) @@ -634,7 +650,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), replicas := replicasPerNode * expectedNodeCount deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), } @@ -662,7 +678,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), env.MeasureDeprovisioningDurationFor(func() { By("kicking off deprovisioning drift by changing the nodeClass AMIFamily") - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} env.ExpectCreatedOrUpdated(nodeClass) env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) @@ -685,7 +701,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), replicas := replicasPerNode * expectedNodeCount deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), } @@ -693,7 +709,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), env.ExpectCreated(deployment) env.EventuallyExpectPendingPodCount(selector, replicas) - var nodes []*v1.Node + var nodes []*corev1.Node env.MeasureProvisioningDurationFor(func() { By("kicking off provisioning by applying the nodePool and nodeClass") env.ExpectCreated(nodePool, nodeClass) diff --git a/test/suites/scale/provisioning_test.go b/test/suites/scale/provisioning_test.go index 79c54896a9b5..edd7a144b14a 100644 --- a/test/suites/scale/provisioning_test.go +++ b/test/suites/scale/provisioning_test.go @@ -21,14 +21,14 @@ import ( "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/labels" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/debug" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" @@ -38,8 +38,8 @@ import ( const testGroup = "provisioning" var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), func() { - var nodePool *corev1beta1.NodePool - var nodeClass *v1beta1.EC2NodeClass + var nodePool *karpv1.NodePool + var nodeClass *v1.EC2NodeClass var deployment *appsv1.Deployment var selector labels.Selector var dsCount int @@ -48,18 +48,22 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu nodeClass = env.DefaultEC2NodeClass() nodePool = env.DefaultNodePool(nodeClass) nodePool.Spec.Limits = nil - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceHypervisor, - Operator: v1.NodeSelectorOpIn, + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{{ + Nodes: "70%", + }} + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceHypervisor, + Operator: corev1.NodeSelectorOpIn, Values: []string{"nitro"}, - }}) + }, + }) deployment = test.Deployment(test.DeploymentOptions{ PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("50Mi"), + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("50Mi"), }, }, TerminationGracePeriodSeconds: lo.ToPtr[int64](0), @@ -85,12 +89,12 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) // Hostname anti-affinity to require one pod on each node - deployment.Spec.Template.Spec.Affinity = &v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + deployment.Spec.Template.Spec.Affinity = &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ { LabelSelector: deployment.Spec.Selector, - TopologyKey: v1.LabelHostname, + TopologyKey: corev1.LabelHostname, }, }, }, @@ -132,22 +136,22 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) // Hostname anti-affinity to require one pod on each node - deployment.Spec.Template.Spec.Affinity = &v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + deployment.Spec.Template.Spec.Affinity = &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ { LabelSelector: deployment.Spec.Selector, - TopologyKey: v1.LabelHostname, + TopologyKey: corev1.LabelHostname, }, }, }, } - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ // minValues is restricted to 30 to have enough instance types to be sent to launch API and not make this test flaky. - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpExists, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpExists, }, MinValues: lo.ToPtr(30), }) @@ -166,7 +170,7 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu env.EventuallyExpectHealthyPodCount(selector, replicas) }, map[string]string{ aws.TestCategoryDimension: testGroup, - aws.TestNameDimension: "node-dense", + aws.TestNameDimension: "node-dense-min-val", aws.ProvisionedNodeCountDimension: strconv.Itoa(expectedNodeCount), aws.DeprovisionedNodeCountDimension: strconv.Itoa(0), aws.PodDensityDimension: strconv.Itoa(replicasPerNode), @@ -178,14 +182,14 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu expectedNodeCount := 60 replicas := replicasPerNode * expectedNodeCount deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), } - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ // With Prefix Delegation enabled, .large instances can have 434 pods. - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceSize, + Operator: corev1.NodeSelectorOpIn, Values: []string{"large"}, }, }, @@ -199,9 +203,6 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu By("kicking off provisioning by applying the nodePool and nodeClass") env.ExpectCreated(nodePool, nodeClass) - env.EventuallyExpectCreatedNodeClaimCount("==", expectedNodeCount) - env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) - env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) }, map[string]string{ aws.TestCategoryDimension: testGroup, @@ -217,23 +218,23 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu expectedNodeCount := 60 replicas := replicasPerNode * expectedNodeCount deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), } test.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ + karpv1.NodeSelectorRequirementWithMinValues{ // With Prefix Delegation enabled, .large instances can have 434 pods. - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceSize, + Operator: corev1.NodeSelectorOpIn, Values: []string{"large"}, }, }, - corev1beta1.NodeSelectorRequirementWithMinValues{ + karpv1.NodeSelectorRequirementWithMinValues{ // minValues is restricted to 30 to have enough instance types to be sent to launch API and not make this test flaky. - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpExists, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpExists, }, MinValues: lo.ToPtr(30), }, @@ -247,13 +248,10 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu By("kicking off provisioning by applying the nodePool and nodeClass") env.ExpectCreated(nodePool, nodeClass) - env.EventuallyExpectCreatedNodeClaimCount("==", expectedNodeCount) - env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) - env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) }, map[string]string{ aws.TestCategoryDimension: testGroup, - aws.TestNameDimension: "pod-dense", + aws.TestNameDimension: "pod-dense-min-val", aws.ProvisionedNodeCountDimension: strconv.Itoa(expectedNodeCount), aws.DeprovisionedNodeCountDimension: strconv.Itoa(0), aws.PodDensityDimension: strconv.Itoa(replicasPerNode), diff --git a/test/suites/scheduling/suite_test.go b/test/suites/scheduling/suite_test.go new file mode 100644 index 000000000000..e0fe0d09c0f7 --- /dev/null +++ b/test/suites/scheduling/suite_test.go @@ -0,0 +1,715 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduling_test + +import ( + "fmt" + "testing" + "time" + + "github.com/awslabs/operatorpkg/object" + "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" + + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + "sigs.k8s.io/karpenter/pkg/test" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + "github.com/aws/karpenter-provider-aws/test/pkg/debug" + environmentaws "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var env *environmentaws.Environment +var nodeClass *v1.EC2NodeClass +var nodePool *karpv1.NodePool + +func TestScheduling(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + env = environmentaws.NewEnvironment(t) + }) + AfterSuite(func() { + env.Stop() + }) + RunSpecs(t, "Scheduling") +} + +var _ = BeforeEach(func() { + env.BeforeEach() + nodeClass = env.DefaultEC2NodeClass() + nodePool = env.DefaultNodePool(nodeClass) +}) +var _ = AfterEach(func() { env.Cleanup() }) +var _ = AfterEach(func() { env.AfterEach() }) +var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { + var selectors sets.Set[string] + + BeforeEach(func() { + // Make the NodePool requirements fully flexible, so we can match well-known label keys + nodePool = test.ReplaceRequirements(nodePool, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpExists, + }, + }, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceGeneration, + Operator: corev1.NodeSelectorOpExists, + }, + }, + ) + }) + BeforeAll(func() { + selectors = sets.New[string]() + }) + AfterAll(func() { + // Ensure that we're exercising all well known labels + Expect(lo.Keys(selectors)).To(ContainElements(append(karpv1.WellKnownLabels.UnsortedList(), lo.Keys(karpv1.NormalizedLabels)...))) + }) + + It("should apply annotations to the node", func() { + nodePool.Spec.Template.Annotations = map[string]string{ + "foo": "bar", + karpv1.DoNotDisruptAnnotationKey: "true", + } + pod := test.Pod() + env.ExpectCreated(nodeClass, nodePool, pod) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + Expect(env.GetNode(pod.Spec.NodeName).Annotations).To(And(HaveKeyWithValue("foo", "bar"), HaveKeyWithValue(karpv1.DoNotDisruptAnnotationKey, "true"))) + }) + + Context("Labels", func() { + It("should support well-known labels for instance type selection", func() { + nodeSelector := map[string]string{ + // Well Known + karpv1.NodePoolLabelKey: nodePool.Name, + corev1.LabelInstanceTypeStable: "c5.large", + // Well Known to AWS + v1.LabelInstanceHypervisor: "nitro", + v1.LabelInstanceCategory: "c", + v1.LabelInstanceGeneration: "5", + v1.LabelInstanceFamily: "c5", + v1.LabelInstanceSize: "large", + v1.LabelInstanceCPU: "2", + v1.LabelInstanceCPUManufacturer: "intel", + v1.LabelInstanceMemory: "4096", + v1.LabelInstanceEBSBandwidth: "4750", + v1.LabelInstanceNetworkBandwidth: "750", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known labels for zone id selection", func() { + selectors.Insert(v1.LabelTopologyZoneID) // Add node selector keys to selectors used in testing to ensure we test all labels + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeRequirements: []corev1.NodeSelectorRequirement{ + { + Key: v1.LabelTopologyZoneID, + Operator: corev1.NodeSelectorOpIn, + Values: []string{env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName})[0].ZoneInfo.ZoneID}, + }, + }, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known labels for local NVME storage", func() { + selectors.Insert(v1.LabelInstanceLocalNVME) // Add node selector keys to selectors used in testing to ensure we test all labels + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodePreferences: []corev1.NodeSelectorRequirement{ + { + Key: v1.LabelInstanceLocalNVME, + Operator: corev1.NodeSelectorOpGt, + Values: []string{"0"}, + }, + }, + NodeRequirements: []corev1.NodeSelectorRequirement{ + { + Key: v1.LabelInstanceLocalNVME, + Operator: corev1.NodeSelectorOpGt, + Values: []string{"0"}, + }, + }, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known labels for encryption in transit", func() { + selectors.Insert(v1.LabelInstanceEncryptionInTransitSupported) // Add node selector keys to selectors used in testing to ensure we test all labels + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodePreferences: []corev1.NodeSelectorRequirement{ + { + Key: v1.LabelInstanceEncryptionInTransitSupported, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"true"}, + }, + }, + NodeRequirements: []corev1.NodeSelectorRequirement{ + { + Key: v1.LabelInstanceEncryptionInTransitSupported, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"true"}, + }, + }, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known deprecated labels", func() { + nodeSelector := map[string]string{ + // Deprecated Labels + corev1.LabelFailureDomainBetaRegion: env.Region, + corev1.LabelFailureDomainBetaZone: fmt.Sprintf("%sa", env.Region), + "topology.ebs.csi.aws.com/zone": fmt.Sprintf("%sa", env.Region), + + "beta.kubernetes.io/arch": "amd64", + "beta.kubernetes.io/os": "linux", + corev1.LabelInstanceType: "c5.large", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known labels for topology and architecture", func() { + nodeSelector := map[string]string{ + // Well Known + karpv1.NodePoolLabelKey: nodePool.Name, + corev1.LabelTopologyRegion: env.Region, + corev1.LabelTopologyZone: fmt.Sprintf("%sa", env.Region), + corev1.LabelOSStable: "linux", + corev1.LabelArchStable: "amd64", + karpv1.CapacityTypeLabelKey: karpv1.CapacityTypeOnDemand, + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known labels for a gpu (nvidia)", func() { + nodeSelector := map[string]string{ + v1.LabelInstanceGPUName: "t4", + v1.LabelInstanceGPUMemory: "16384", + v1.LabelInstanceGPUManufacturer: "nvidia", + v1.LabelInstanceGPUCount: "1", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + // Use AL2 AMIs instead of AL2023 since accelerated AMIs aren't yet available + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known labels for an accelerator (inferentia)", func() { + nodeSelector := map[string]string{ + v1.LabelInstanceAcceleratorName: "inferentia", + v1.LabelInstanceAcceleratorManufacturer: "aws", + v1.LabelInstanceAcceleratorCount: "1", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + // Use AL2 AMIs instead of AL2023 since accelerated AMIs aren't yet available + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) + }) + // Windows tests are can flake due to the instance types that are used in testing. + // The VPC Resource controller will need to support the instance types that are used. + // If the instance type is not supported by the controller resource `vpc.amazonaws.com/PrivateIPv4Address` will not register. + // Issue: https://github.com/aws/karpenter-provider-aws/issues/4472 + // See: https://github.com/aws/amazon-vpc-resource-controller-k8s/blob/master/pkg/aws/vpc/limits.go + It("should support well-known labels for windows-build version", func() { + env.ExpectWindowsIPAMEnabled() + DeferCleanup(func() { + env.ExpectWindowsIPAMDisabled() + }) + + nodeSelector := map[string]string{ + // Well Known + corev1.LabelWindowsBuild: v1.Windows2022Build, + corev1.LabelOSStable: string(corev1.Windows), // Specify the OS to enable vpc-resource-controller to inject the PrivateIPv4Address resource + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + Image: environmentaws.WindowsDefaultImage, + }}) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "windows2022@latest"}} + test.ReplaceRequirements(nodePool, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Windows)}, + }, + }, + ) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCountWithTimeout(time.Minute*15, labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) + }) + DescribeTable("should support restricted label domain exceptions", func(domain string) { + // Assign labels to the nodepool so that it has known values + test.ReplaceRequirements(nodePool, + karpv1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: domain + "/team", Operator: corev1.NodeSelectorOpExists}}, + karpv1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: domain + "/custom-label", Operator: corev1.NodeSelectorOpExists}}, + karpv1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: "subdomain." + domain + "/custom-label", Operator: corev1.NodeSelectorOpExists}}, + ) + nodeSelector := map[string]string{ + domain + "/team": "team-1", + domain + "/custom-label": "custom-value", + "subdomain." + domain + "/custom-label": "custom-value", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + node := env.ExpectCreatedNodeCount("==", 1)[0] + // Ensure that the requirements/labels specified above are propagated onto the node + for k, v := range nodeSelector { + Expect(node.Labels).To(HaveKeyWithValue(k, v)) + } + }, + Entry("node-restriction.kuberentes.io", "node-restriction.kuberentes.io"), + Entry("node.kubernetes.io", "node.kubernetes.io"), + Entry("kops.k8s.io", "kops.k8s.io"), + ) + }) + + Context("Provisioning", func() { + It("should provision a node for naked pods", func() { + pod := test.Pod() + + env.ExpectCreated(nodeClass, nodePool, pod) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + }) + It("should provision a node for a deployment", Label(debug.NoWatch), Label(debug.NoEvents), func() { + deployment := test.Deployment(test.DeploymentOptions{Replicas: 50}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("<=", 2) // should probably all land on a single node, but at worst two depending on batching + }) + It("should provision a node for a self-affinity deployment", func() { + // just two pods as they all need to land on the same node + podLabels := map[string]string{"test": "self-affinity"} + deployment := test.Deployment(test.DeploymentOptions{ + Replicas: 2, + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + PodRequirements: []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{MatchLabels: podLabels}, + TopologyKey: corev1.LabelHostname, + }, + }, + }, + }) + + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), 2) + env.ExpectCreatedNodeCount("==", 1) + }) + It("should provision three nodes for a zonal topology spread", func() { + // one pod per zone + podLabels := map[string]string{"test": "zonal-spread"} + deployment := test.Deployment(test.DeploymentOptions{ + Replicas: 3, + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: corev1.LabelTopologyZone, + WhenUnsatisfiable: corev1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: podLabels}, + MinDomains: lo.ToPtr(int32(3)), + }, + }, + }, + }) + + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(podLabels), 3) + // Karpenter will launch three nodes, however if all three nodes don't get register with the cluster at the same time, two pods will be placed on one node. + // This can result in a case where all 3 pods are healthy, while there are only two created nodes. + // In that case, we still expect to eventually have three nodes. + env.EventuallyExpectNodeCount("==", 3) + }) + It("should provision a node using a NodePool with higher priority", func() { + nodePoolLowPri := test.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Weight: lo.ToPtr(int32(10)), + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Linux)}, + }, + }, + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"t3.nano"}, + }, + }, + }, + }, + }, + }, + }) + nodePoolHighPri := test.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Weight: lo.ToPtr(int32(100)), + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Linux)}, + }, + }, + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"c5.large"}, + }, + }, + }, + }, + }, + }, + }) + pod := test.Pod() + env.ExpectCreated(pod, nodeClass, nodePoolLowPri, nodePoolHighPri) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + Expect(lo.FromPtr(env.GetInstance(pod.Spec.NodeName).InstanceType)).To(Equal("c5.large")) + Expect(env.GetNode(pod.Spec.NodeName).Labels[karpv1.NodePoolLabelKey]).To(Equal(nodePoolHighPri.Name)) + }) + + DescribeTable( + "should provision a right-sized node when a pod has InitContainers (cpu)", + func(expectedNodeCPU string, containerRequirements corev1.ResourceRequirements, initContainers ...corev1.Container) { + if env.K8sMinorVersion() < 29 { + Skip("native sidecar containers are only enabled on EKS 1.29+") + } + + labels := map[string]string{"test": test.RandomName()} + // Create a buffer pod to even out the total resource requests regardless of the daemonsets on the cluster. Assumes + // CPU is the resource in contention and that total daemonset CPU requests <= 3. + dsBufferPod := test.Pod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + PodRequirements: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: corev1.LabelHostname, + }}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: func() resource.Quantity { + dsOverhead := env.GetDaemonSetOverhead(nodePool) + base := lo.ToPtr(resource.MustParse("3")) + base.Sub(*dsOverhead.Cpu()) + return *base + }(), + }, + }, + }) + + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCPU, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"4", "8"}, + }, + }, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"t"}, + }, + }) + pod := test.Pod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + PodRequirements: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: corev1.LabelHostname, + }}, + InitContainers: initContainers, + ResourceRequirements: containerRequirements, + }) + env.ExpectCreated(nodePool, nodeClass, dsBufferPod, pod) + env.EventuallyExpectHealthy(pod) + node := env.ExpectCreatedNodeCount("==", 1)[0] + Expect(node.ObjectMeta.GetLabels()[v1.LabelInstanceCPU]).To(Equal(expectedNodeCPU)) + }, + Entry("sidecar requirements + later init requirements do exceed container requirements", "8", corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("400m")}, + }, ephemeralInitContainer(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("300m")}, + }), corev1.Container{ + RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("350m")}, + }, + }, ephemeralInitContainer(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + })), + Entry("sidecar requirements + later init requirements do not exceed container requirements", "4", corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("400m")}, + }, ephemeralInitContainer(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("300m")}, + }), corev1.Container{ + RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("350m")}, + }, + }, ephemeralInitContainer(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("300m")}, + })), + Entry("init container requirements exceed all later requests", "8", corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("400m")}, + }, corev1.Container{ + RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + }, + }, ephemeralInitContainer(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1500m")}, + }), corev1.Container{ + RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + }, + }), + ) + It("should provision a right-sized node when a pod has InitContainers (mixed resources)", func() { + if env.K8sMinorVersion() < 29 { + Skip("native sidecar containers are only enabled on EKS 1.29+") + } + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1.LabelInstanceCategory, + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"t"}, + }, + }) + pod := test.Pod(test.PodOptions{ + InitContainers: []corev1.Container{ + { + RestartPolicy: lo.ToPtr(corev1.ContainerRestartPolicyAlways), + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }}, + }, + ephemeralInitContainer(corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("4Gi"), + }}), + }, + ResourceRequirements: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }}, + }) + env.ExpectCreated(nodePool, nodeClass, pod) + env.EventuallyExpectHealthy(pod) + }) + + It("should provision a node for a pod with overlapping zone and zone-id requirements", func() { + subnetInfo := lo.UniqBy(env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName}), func(s environmentaws.SubnetInfo) string { + return s.Zone + }) + Expect(len(subnetInfo)).To(BeNumerically(">=", 3)) + + // Create a pod with 'overlapping' zone and zone-id requirements. With two options for each label, but only one pair of zone-zoneID that maps to the + // same AZ, we will always expect the pod to be scheduled to that AZ. In this case, this is the mapping at zone[1]. + pod := test.Pod(test.PodOptions{ + NodeRequirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelTopologyZone, + Operator: corev1.NodeSelectorOpIn, + Values: lo.Map(subnetInfo[0:2], func(info environmentaws.SubnetInfo, _ int) string { return info.Zone }), + }, + { + Key: v1.LabelTopologyZoneID, + Operator: corev1.NodeSelectorOpIn, + Values: lo.Map(subnetInfo[1:3], func(info environmentaws.SubnetInfo, _ int) string { return info.ZoneID }), + }, + }, + }) + env.ExpectCreated(nodePool, nodeClass, pod) + node := env.EventuallyExpectInitializedNodeCount("==", 1)[0] + Expect(node.Labels[corev1.LabelTopologyZone]).To(Equal(subnetInfo[1].Zone)) + Expect(node.Labels[v1.LabelTopologyZoneID]).To(Equal(subnetInfo[1].ZoneID)) + }) + It("should provision nodes for pods with zone-id requirements in the correct zone", func() { + // Each pod specifies a requirement on this expected zone, where the value is the matching zone for the + // required zone-id. This allows us to verify that Karpenter launched the node in the correct zone, even if + // it doesn't add the zone-id label and the label is added by CCM. If we didn't take this approach, we would + // succeed even if Karpenter doesn't add the label and /or incorrectly generated offerings on k8s 1.30 and + // above. This is an unlikely scenario, and adding this check is a defense in depth measure. + const expectedZoneLabel = "expected-zone-label" + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: expectedZoneLabel, + Operator: corev1.NodeSelectorOpExists, + }, + }) + + subnetInfo := lo.UniqBy(env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName}), func(s environmentaws.SubnetInfo) string { + return s.Zone + }) + pods := lo.Map(subnetInfo, func(info environmentaws.SubnetInfo, _ int) *corev1.Pod { + return test.Pod(test.PodOptions{ + NodeRequirements: []corev1.NodeSelectorRequirement{ + { + Key: expectedZoneLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{info.Zone}, + }, + { + Key: v1.LabelTopologyZoneID, + Operator: corev1.NodeSelectorOpIn, + Values: []string{info.ZoneID}, + }, + }, + }) + }) + + env.ExpectCreated(nodePool, nodeClass) + for _, pod := range pods { + env.ExpectCreated(pod) + } + nodes := env.EventuallyExpectInitializedNodeCount("==", len(subnetInfo)) + for _, node := range nodes { + expectedZone, ok := node.Labels[expectedZoneLabel] + Expect(ok).To(BeTrue()) + Expect(node.Labels[corev1.LabelTopologyZone]).To(Equal(expectedZone)) + zoneInfo, ok := lo.Find(subnetInfo, func(info environmentaws.SubnetInfo) bool { + return info.Zone == expectedZone + }) + Expect(ok).To(BeTrue()) + Expect(node.Labels[v1.LabelTopologyZoneID]).To(Equal(zoneInfo.ZoneID)) + } + }) + }) +}) + +func ephemeralInitContainer(requirements corev1.ResourceRequirements) corev1.Container { + return corev1.Container{ + Image: environmentaws.EphemeralInitContainerImage, + Command: []string{"/bin/sh"}, + Args: []string{"-c", "sleep 5"}, + Resources: requirements, + } +} diff --git a/test/suites/integration/storage_test.go b/test/suites/storage/suite_test.go similarity index 55% rename from test/suites/integration/storage_test.go rename to test/suites/storage/suite_test.go index 8c6f4b5e781f..fca54d99416f 100644 --- a/test/suites/integration/storage_test.go +++ b/test/suites/storage/suite_test.go @@ -12,30 +12,61 @@ See the License for the specific language governing permissions and limitations under the License. */ -package integration_test +package storage_test import ( "fmt" "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/labels" + + awssdk "github.com/aws/aws-sdk-go/aws" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + environmentaws "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/aws/aws-sdk-go/aws" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/samber/lo" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/errors" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/karpenter/pkg/test" ) +var env *environmentaws.Environment +var nodeClass *v1.EC2NodeClass +var nodePool *karpv1.NodePool + +func TestStorage(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + env = environmentaws.NewEnvironment(t) + }) + AfterSuite(func() { + env.Stop() + }) + RunSpecs(t, "Storage") +} + +var _ = BeforeEach(func() { + env.BeforeEach() + nodeClass = env.DefaultEC2NodeClass() + nodePool = env.DefaultNodePool(nodeClass) +}) +var _ = AfterEach(func() { env.Cleanup() }) +var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Persistent Volumes", func() { Context("Static", func() { It("should run a pod with a pre-bound persistent volume (empty storage class)", func() { @@ -127,7 +158,7 @@ var _ = Describe("Persistent Volumes", func() { ObjectMeta: metav1.ObjectMeta{ Name: "test-storage-class", }, - Provisioner: aws.String("ebs.csi.aws.com"), + Provisioner: awssdk.String("ebs.csi.aws.com"), VolumeBindingMode: lo.ToPtr(storagev1.VolumeBindingWaitForFirstConsumer), }) }) @@ -148,8 +179,8 @@ var _ = Describe("Persistent Volumes", func() { subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": env.ClusterName}) shuffledAZs := lo.Shuffle(lo.Keys(subnets)) - storageClass.AllowedTopologies = []v1.TopologySelectorTerm{{ - MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{{ + storageClass.AllowedTopologies = []corev1.TopologySelectorTerm{{ + MatchLabelExpressions: []corev1.TopologySelectorLabelRequirement{{ Key: "topology.ebs.csi.aws.com/zone", Values: []string{shuffledAZs[0]}, }}, @@ -173,12 +204,12 @@ var _ = Describe("Persistent Volumes", func() { }) count := 2 - pvcs := lo.Times(count, func(_ int) *v1.PersistentVolumeClaim { + pvcs := lo.Times(count, func(_ int) *corev1.PersistentVolumeClaim { return test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ StorageClassName: &storageClass.Name, }) }) - pods := lo.Map(pvcs, func(pvc *v1.PersistentVolumeClaim, _ int) *v1.Pod { + pods := lo.Map(pvcs, func(pvc *corev1.PersistentVolumeClaim, _ int) *corev1.Pod { return test.Pod(test.PodOptions{ PersistentVolumeClaims: []string{pvc.Name}, }) @@ -208,14 +239,143 @@ var _ = Describe("Persistent Volumes", func() { }) }) +var _ = Describe("Stateful workloads", func() { + var numPods int + var persistentVolumeClaim *corev1.PersistentVolumeClaim + var storageClass *storagev1.StorageClass + var statefulSet *appsv1.StatefulSet + var selector labels.Selector + BeforeEach(func() { + // Ensure that the EBS driver is installed, or we can't run the test. + var ds appsv1.DaemonSet + if err := env.Client.Get(env.Context, client.ObjectKey{ + Namespace: "kube-system", + Name: "ebs-csi-node", + }, &ds); err != nil { + if errors.IsNotFound(err) { + Skip(fmt.Sprintf("skipping StatefulSet test due to missing EBS driver %s", err)) + } else { + Fail(fmt.Sprintf("determining EBS driver status, %s", err)) + } + } + + numPods = 1 + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": env.ClusterName}) + shuffledAZs := lo.Shuffle(lo.Keys(subnets)) + + storageClass = test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-storage-class", + }, + Provisioner: awssdk.String("ebs.csi.aws.com"), + VolumeBindingMode: lo.ToPtr(storagev1.VolumeBindingWaitForFirstConsumer), + }) + + storageClass.AllowedTopologies = []corev1.TopologySelectorTerm{{ + MatchLabelExpressions: []corev1.TopologySelectorLabelRequirement{{ + Key: "topology.ebs.csi.aws.com/zone", + Values: []string{shuffledAZs[0]}, + }}, + }} + + persistentVolumeClaim = test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + StorageClassName: &storageClass.Name, + }) + statefulSet = test.StatefulSet(test.StatefulSetOptions{ + Replicas: int32(numPods), + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "my-app", + }}, + }, + }) + // Ensure same volume is used across replica restarts. + statefulSet.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{*persistentVolumeClaim} + // Ensure volume mounts to pod, so that we test that we avoid the 6+ minute force detach delay. + vm := corev1.VolumeMount{ + Name: persistentVolumeClaim.Name, + MountPath: "/usr/share", + } + statefulSet.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{vm} + selector = labels.SelectorFromSet(statefulSet.Spec.Selector.MatchLabels) + }) + + It("should run on a new node without 6+ minute delays when disrupted", func() { + // EBS volume detach + attach should usually take ~20s. Extra time is to prevent flakes due to EC2 APIs. + forceDetachTimeout := 2 * time.Minute + + env.ExpectCreated(nodeClass, nodePool, storageClass, statefulSet) + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] + env.EventuallyExpectHealthyPodCount(selector, numPods) + + env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration + + // Delete original nodeClaim to get the original node deleted + env.ExpectDeleted(nodeClaim) + + // Eventually the node will be tainted, which means its actively being disrupted + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).Should(Succeed()) + _, ok := lo.Find(node.Spec.Taints, func(t corev1.Taint) bool { + return karpv1.IsDisruptingTaint(t) + }) + g.Expect(ok).To(BeTrue()) + }).Should(Succeed()) + + env.EventuallyExpectCreatedNodeCount(">=", 1) + + // After the deletion timestamp is set and all pods are drained the node should be gone. + env.EventuallyExpectNotFound(nodeClaim, node) + + // We expect the stateful workload to become healthy on new node before the 6-minute force detach timeout. + // We start timer after pod binds to node because volume attachment happens during ContainerCreating + env.EventuallyExpectCreatedNodeClaimCount("==", 1) + env.EventuallyExpectCreatedNodeCount(">=", 1) + env.EventuallyExpectBoundPodCount(selector, numPods) + env.EventuallyExpectHealthyPodCountWithTimeout(forceDetachTimeout, selector, numPods) + }) + It("should not block node deletion if stateful workload cannot be drained", func() { + // Make pod un-drain-able by tolerating disruption taint. + statefulSet.Spec.Template.Spec.Tolerations = []corev1.Toleration{{ + Key: "karpenter.sh/disruption", + Operator: corev1.TolerationOpEqual, + Value: "disrupting", + Effect: corev1.TaintEffectNoExecute, + }} + + env.ExpectCreated(nodeClass, nodePool, storageClass, statefulSet) + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] + env.EventuallyExpectHealthyPodCount(selector, numPods) + + // Delete original nodeClaim to get the original node deleted + env.ExpectDeleted(nodeClaim) + + // Eventually the node will be tainted, which means its actively being disrupted + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).Should(Succeed()) + _, ok := lo.Find(node.Spec.Taints, func(t corev1.Taint) bool { + return karpv1.IsDisruptingTaint(t) + }) + g.Expect(ok).To(BeTrue()) + }).Should(Succeed()) + + // After the deletion timestamp is set and all pods are drained + // the node should be gone regardless of orphaned volume attachment objects. + env.EventuallyExpectNotFound(nodeClaim, node) + }) +}) + var _ = Describe("Ephemeral Storage", func() { It("should run a pod with instance-store ephemeral storage that exceeds EBS root block device mappings", func() { - nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1beta1.InstanceStorePolicyRAID0) + nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1.InstanceStorePolicyRAID0) pod := test.Pod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceEphemeralStorage: resource.MustParse("100Gi"), + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceEphemeralStorage: resource.MustParse("100Gi"), }, }, }) diff --git a/test/suites/integration/emptiness_test.go b/test/suites/termination/emptiness_test.go similarity index 82% rename from test/suites/integration/emptiness_test.go rename to test/suites/termination/emptiness_test.go index a4aca9792b0d..cf53b749c26b 100644 --- a/test/suites/integration/emptiness_test.go +++ b/test/suites/termination/emptiness_test.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package integration_test +package termination_test import ( "fmt" @@ -22,14 +22,13 @@ import ( appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" ) @@ -38,8 +37,8 @@ var _ = Describe("Emptiness", func() { var selector labels.Selector var numPods int BeforeEach(func() { - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenEmpty - nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Duration(0))} + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmpty + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") numPods = 1 dep = test.Deployment(test.DeploymentOptions{ @@ -55,7 +54,7 @@ var _ = Describe("Emptiness", func() { Context("Budgets", func() { It("should not allow emptiness if the budget is fully blocking", func() { // We're going to define a budget that doesn't allow any emptiness disruption to happen - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{{ Nodes: "0", }} @@ -68,7 +67,7 @@ var _ = Describe("Emptiness", func() { // Delete the deployment so there is nothing running on the node env.ExpectDeleted(dep) - env.EventuallyExpectEmpty(nodeClaim) + env.EventuallyExpectConsolidatable(nodeClaim) env.ConsistentlyExpectNoDisruptions(1, time.Minute) }) It("should not allow emptiness if the budget is fully blocking during a scheduled time", func() { @@ -77,7 +76,7 @@ var _ = Describe("Emptiness", func() { // the current time and extends 15 minutes past the current time // Times need to be in UTC since the karpenter containers were built in UTC time windowStart := time.Now().Add(-time.Minute * 15).UTC() - nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + nodePool.Spec.Disruption.Budgets = []karpv1.Budget{{ Nodes: "0", Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())), Duration: &metav1.Duration{Duration: time.Minute * 30}, @@ -92,12 +91,12 @@ var _ = Describe("Emptiness", func() { // Delete the deployment so there is nothing running on the node env.ExpectDeleted(dep) - env.EventuallyExpectEmpty(nodeClaim) + env.EventuallyExpectConsolidatable(nodeClaim) env.ConsistentlyExpectNoDisruptions(1, time.Minute) }) }) It("should terminate an empty node", func() { - nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Hour * 300)} + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("10s") const numPods = 1 deployment := test.Deployment(test.DeploymentOptions{Replicas: numPods}) @@ -110,13 +109,13 @@ var _ = Describe("Emptiness", func() { By("making the nodeclaim empty") persisted := deployment.DeepCopy() - deployment.Spec.Replicas = ptr.Int32(0) + deployment.Spec.Replicas = lo.ToPtr(int32(0)) Expect(env.Client.Patch(env, deployment, client.StrategicMergeFrom(persisted))).To(Succeed()) - env.EventuallyExpectEmpty(nodeClaim) + env.EventuallyExpectConsolidatable(nodeClaim) By("waiting for the nodeclaim to deprovision when past its ConsolidateAfter timeout of 0") - nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Duration(0))} + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") env.ExpectUpdated(nodePool) env.EventuallyExpectNotFound(nodeClaim, node) diff --git a/test/suites/termination/suite_test.go b/test/suites/termination/suite_test.go new file mode 100644 index 000000000000..9a9467997807 --- /dev/null +++ b/test/suites/termination/suite_test.go @@ -0,0 +1,51 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package termination_test + +import ( + "testing" + + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var env *aws.Environment +var nodeClass *v1.EC2NodeClass +var nodePool *karpv1.NodePool + +func TestExpiration(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + env = aws.NewEnvironment(t) + }) + AfterSuite(func() { + env.Stop() + }) + RunSpecs(t, "Termination") +} + +var _ = BeforeEach(func() { + env.BeforeEach() + nodeClass = env.DefaultEC2NodeClass() + nodePool = env.DefaultNodePool(nodeClass) +}) + +var _ = AfterEach(func() { env.Cleanup() }) +var _ = AfterEach(func() { env.AfterEach() }) diff --git a/test/suites/termination/termination_grace_period_test.go b/test/suites/termination/termination_grace_period_test.go new file mode 100644 index 000000000000..941297c35555 --- /dev/null +++ b/test/suites/termination/termination_grace_period_test.go @@ -0,0 +1,98 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package termination_test + +import ( + "time" + + coretest "sigs.k8s.io/karpenter/pkg/test" + + "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("TerminationGracePeriod", func() { + BeforeEach(func() { + nodePool.Spec.Template.Spec.TerminationGracePeriod = &metav1.Duration{Duration: time.Second * 30} + }) + It("should delete pod with do-not-disrupt when it reaches its terminationGracePeriodSeconds", func() { + pod := coretest.UnschedulablePod(coretest.PodOptions{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + karpv1.DoNotDisruptAnnotationKey: "true", + }}, TerminationGracePeriodSeconds: lo.ToPtr(int64(15))}) + env.ExpectCreated(nodeClass, nodePool, pod) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] + env.EventuallyExpectHealthy(pod) + + // Delete the nodeclaim to start the TerminationGracePeriod + env.ExpectDeleted(nodeClaim) + + // Eventually the node will be tainted + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).Should(Succeed()) + _, ok := lo.Find(node.Spec.Taints, func(t corev1.Taint) bool { + return karpv1.IsDisruptingTaint(t) + }) + g.Expect(ok).To(BeTrue()) + }).WithTimeout(3 * time.Second).WithPolling(100 * time.Millisecond).Should(Succeed()) + + // Check that pod remains healthy until termination grace period + // subtract the polling time of the eventually above to reduce any races. + env.ConsistentlyExpectHealthyPods(time.Second*15-100*time.Millisecond, pod) + // Both nodeClaim and node should be gone once terminationGracePeriod is reached + env.EventuallyExpectNotFound(nodeClaim, node, pod) + }) + It("should delete pod that has a pre-stop hook after termination grace period seconds", func() { + pod := coretest.UnschedulablePod(coretest.PodOptions{ + PreStopSleep: lo.ToPtr(int64(300)), + TerminationGracePeriodSeconds: lo.ToPtr(int64(15)), + Image: "alpine:3.20.2", + Command: []string{"/bin/sh", "-c", "sleep 30"}}) + env.ExpectCreated(nodeClass, nodePool, pod) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] + env.EventuallyExpectHealthy(pod) + + // Delete the nodeclaim to start the TerminationGracePeriod + env.ExpectDeleted(nodeClaim) + + // Eventually the node will be tainted + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).Should(Succeed()) + _, ok := lo.Find(node.Spec.Taints, func(t corev1.Taint) bool { + return karpv1.IsDisruptingTaint(t) + }) + g.Expect(ok).To(BeTrue()) + }).WithTimeout(3 * time.Second).WithPolling(100 * time.Millisecond).Should(Succeed()) + + env.EventuallyExpectTerminating(pod) + + // Check that pod remains healthy until termination grace period + // subtract the polling time of the eventually above to reduce any races. + env.ConsistentlyExpectTerminatingPods(time.Second*15-100*time.Millisecond, pod) + + // Both nodeClaim and node should be gone once terminationGracePeriod is reached + env.EventuallyExpectNotFound(nodeClaim, node, pod) + }) +}) diff --git a/test/suites/integration/termination_test.go b/test/suites/termination/termination_test.go similarity index 95% rename from test/suites/integration/termination_test.go rename to test/suites/termination/termination_test.go index cbd32fb51a27..7a48fdeee27d 100644 --- a/test/suites/integration/termination_test.go +++ b/test/suites/termination/termination_test.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package integration_test +package termination_test import ( "time" @@ -44,7 +44,7 @@ var _ = Describe("Termination", func() { env.ExpectDeleted(nodes[0]) env.EventuallyExpectNotFound(nodes[0]) Eventually(func(g Gomega) { - g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(Equal("shutting-down")) + g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(BeElementOf("terminated", "shutting-down")) }, time.Second*10).Should(Succeed()) }) }) diff --git a/tools/allocatable-diff/go.mod b/tools/allocatable-diff/go.mod index 8bcdeb37a91f..7192721ad39d 100644 --- a/tools/allocatable-diff/go.mod +++ b/tools/allocatable-diff/go.mod @@ -1,20 +1,22 @@ module github.com/aws/karpenter-provider-aws/tools/allocatable-diff -go 1.22 +go 1.22.4 require ( - github.com/aws/karpenter-provider-aws v0.33.1-0.20231206223517-f73ccfa65419 + github.com/aws/karpenter-provider-aws v0.37.1-0.20240607201537-69a807081c7f github.com/samber/lo v1.39.0 - k8s.io/api v0.28.4 - k8s.io/client-go v0.28.4 - sigs.k8s.io/controller-runtime v0.16.3 - sigs.k8s.io/karpenter v0.33.0 + k8s.io/api v0.30.1 + k8s.io/client-go v0.30.1 + sigs.k8s.io/controller-runtime v0.18.3 + sigs.k8s.io/karpenter v0.37.1-0.20240605225346-c7c5068db687 ) require ( contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect - github.com/aws/aws-sdk-go v1.48.13 // indirect + github.com/aws/aws-sdk-go v1.53.14 // indirect + github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647 // indirect + github.com/awslabs/operatorpkg v0.0.0-20240605172541-88cf99023fa4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/blendle/zapdriver v1.3.1 // indirect @@ -22,11 +24,11 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch/v5 v5.7.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -34,11 +36,11 @@ require ( github.com/gobuffalo/flect v1.0.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -48,32 +50,32 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.53.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/prometheus/statsd_exporter v0.24.0 // indirect - github.com/spf13/cobra v1.7.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.opencensus.io v0.24.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.26.0 // indirect + go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sync v0.5.0 // indirect - golang.org/x/sys v0.14.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.5.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/api v0.146.0 // indirect @@ -85,16 +87,16 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.28.4 // indirect - k8s.io/apimachinery v0.28.4 // indirect - k8s.io/cloud-provider v0.28.4 // indirect - k8s.io/component-base v0.28.4 // indirect - k8s.io/csi-translation-lib v0.28.4 // indirect - k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/apimachinery v0.30.1 // indirect + k8s.io/cloud-provider v0.30.1 // indirect + k8s.io/component-base v0.30.1 // indirect + k8s.io/csi-translation-lib v0.30.1 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/tools/allocatable-diff/go.sum b/tools/allocatable-diff/go.sum index 5a614287ebfd..fd3ad24c358b 100644 --- a/tools/allocatable-diff/go.sum +++ b/tools/allocatable-diff/go.sum @@ -48,10 +48,14 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8V github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= -github.com/aws/aws-sdk-go v1.48.13 h1:6N4GTme6MpxfCisWf5pql8k3TBORiKTmbeutZCDXlG8= -github.com/aws/aws-sdk-go v1.48.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/aws/karpenter-provider-aws v0.33.1-0.20231206223517-f73ccfa65419 h1:zejIHIPkRwfw1VFHFqtM1YHYcxH3vcBkQIQbydw6YjI= -github.com/aws/karpenter-provider-aws v0.33.1-0.20231206223517-f73ccfa65419/go.mod h1:PYMJm2lX/ppJjEfQg3OEeMb8UyYXW762mlmbJ7WueEg= +github.com/aws/aws-sdk-go v1.53.14 h1:SzhkC2Pzag0iRW8WBb80RzKdGXDydJR9LAMs2GyKJ2M= +github.com/aws/aws-sdk-go v1.53.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/karpenter-provider-aws v0.37.1-0.20240607201537-69a807081c7f h1:cPOalRi92kdfw90VLOchNzwQ3tkeVFXS01J0rrfncCg= +github.com/aws/karpenter-provider-aws v0.37.1-0.20240607201537-69a807081c7f/go.mod h1:wooDcsc/zXykRD/xK+aUvhxVcyJWAQU8qQ/z5wqlYlI= +github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647 h1:8yRBVsjGmI7qQsPWtIrbWP+XfwHO9Wq7gdLVzjqiZFs= +github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647/go.mod h1:9NafTAUHL0FlMeL6Cu5PXnMZ1q/LnC9X2emLXHsVbM8= +github.com/awslabs/operatorpkg v0.0.0-20240605172541-88cf99023fa4 h1:EVFVrteX0PQuofO9Ah4rf4aGyUkBM3lLuKgzwilAEAg= +github.com/awslabs/operatorpkg v0.0.0-20240605172541-88cf99023fa4/go.mod h1:OR0NDOTl6XUXKgcksUab5d7mCnpaZf7Ko4eWEbheJTY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -72,7 +76,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -85,10 +89,10 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc= -github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -105,8 +109,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -119,7 +123,8 @@ github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogB github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -156,8 +161,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= @@ -188,12 +193,12 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= -github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= @@ -242,8 +247,6 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -257,14 +260,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs= -github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM= -github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= -github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= -github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -280,22 +283,22 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= +github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -307,6 +310,8 @@ github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3c github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= github.com/prometheus/statsd_exporter v0.24.0 h1:aZmN6CzS2H1Non1JKZdjkQlAkDtGoQBYIESk2SlU1OI= github.com/prometheus/statsd_exporter v0.24.0/go.mod h1:+dQiRTqn9DnPmN5mI5Xond+k8nuRKzdgh1omxh9OgFY= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -317,14 +322,15 @@ github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXn github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -332,8 +338,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -351,14 +358,14 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -399,8 +406,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -437,8 +444,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -446,8 +453,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= -golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -460,8 +467,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -504,13 +511,12 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -519,8 +525,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -569,8 +575,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -700,38 +706,40 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY= -k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0= -k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU= -k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM= -k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8= -k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg= -k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY= -k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4= -k8s.io/cloud-provider v0.28.4 h1:7obmeuJJ5CYTO9HANDqemf/d2v95U+F0t8aeH4jNOsQ= -k8s.io/cloud-provider v0.28.4/go.mod h1:xbhmGZ7wRHgXFP3SNsvdmFRO87KJIvirDYQA5ydMgGA= -k8s.io/component-base v0.28.4 h1:c/iQLWPdUgI90O+T9TeECg8o7N3YJTiuz2sKxILYcYo= -k8s.io/component-base v0.28.4/go.mod h1:m9hR0uvqXDybiGL2nf/3Lf0MerAfQXzkfWhUY58JUbU= -k8s.io/csi-translation-lib v0.28.4 h1:4TrU2zefZGU5HQCyPZvcPxkS6IowqZ/jBs2Qi/dPUpc= -k8s.io/csi-translation-lib v0.28.4/go.mod h1:oxwDdx0hyVqViINOUF7TGrVt51eqsOkQ0BTI+A9QcQs= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY= +k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM= +k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xarePws= +k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4= +k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U= +k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q= +k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc= +k8s.io/cloud-provider v0.30.1 h1:OslHpog97zG9Kr7/vV1ki8nLKq8xTPUkN/kepCxBqKI= +k8s.io/cloud-provider v0.30.1/go.mod h1:1uZp+FSskXQoeAAIU91/XCO8X/9N1U3z5usYeSLT4MI= +k8s.io/component-base v0.30.1 h1:bvAtlPh1UrdaZL20D9+sWxsJljMi0QZ3Lmw+kmZAaxQ= +k8s.io/component-base v0.30.1/go.mod h1:e/X9kDiOebwlI41AvBHuWdqFriSRrX50CdwA9TFaHLI= +k8s.io/csi-translation-lib v0.30.1 h1:fIBtNMQjyr7HFv3xGSSH9cWOQS1K1kIBmZ1zRsHuVKs= +k8s.io/csi-translation-lib v0.30.1/go.mod h1:l0HrIBIxUKRvqnNWqn6AXTYgUa2mAFLT6bjo1lU+55U= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/kubernetes v1.30.1 h1:XlqS6KslLEA5mQzLK2AJrhr4Z1m8oJfkhHiWJ5lue+I= +k8s.io/kubernetes v1.30.1/go.mod h1:yPbIk3MhmhGigX62FLJm+CphNtjxqCvAIFQXup6RKS0= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd h1:KJXBX9dOmRTUWduHg1gnWtPGIEl+GMh8UHdrBEZgOXE= knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd/go.mod h1:36cYnaOVHkzmhgybmYX6zDaTl3PakFeJQJl7wi6/RLE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= -sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= +sigs.k8s.io/controller-runtime v0.18.3 h1:B5Wmmo8WMWK7izei+2LlXLVDGzMwAHBNLX68lwtlSR4= +sigs.k8s.io/controller-runtime v0.18.3/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/karpenter v0.33.0 h1:FvFzhYMwFackafImFeFz/ti/z/55NiZtr8HXiS+z9lU= -sigs.k8s.io/karpenter v0.33.0/go.mod h1:7hPB7kdImFHAFp7pqPUqgBDOrh8GEcRnHT5uIlsXMKA= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/karpenter v0.37.1-0.20240605225346-c7c5068db687 h1:I4JBstyyyAwKDXJsb86HAQRPvR2VowsqF81F734L2G8= +sigs.k8s.io/karpenter v0.37.1-0.20240605225346-c7c5068db687/go.mod h1:SO2WU8Li+bwkLiiEYrRmuhP0QUseZCyYgo6OoXgd9uk= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/tools/allocatable-diff/main.go b/tools/allocatable-diff/main.go index 0a675056025a..10990107aef7 100644 --- a/tools/allocatable-diff/main.go +++ b/tools/allocatable-diff/main.go @@ -78,7 +78,6 @@ func main() { op.GetClient(), op.AMIProvider, op.SecurityGroupProvider, - op.SubnetProvider, ) instanceTypes := lo.Must(cloudProvider.GetInstanceTypes(ctx, nil)) diff --git a/website/content/en/docs/concepts/_index.md b/website/content/en/docs/concepts/_index.md index 67d573135369..e7f8ce4cb500 100755 --- a/website/content/en/docs/concepts/_index.md +++ b/website/content/en/docs/concepts/_index.md @@ -29,7 +29,7 @@ Once privileges are in place, Karpenter is deployed with a Helm chart. ### Configuring NodePools -Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. To configure Karpenter, you create [NodePools]({{}}) that define how Karpenter manages unschedulable pods and configures nodes. You will also define behaviors for your NodePools, capturing details like how Karpenter handles disruption of nodes and setting limits and weights for each NodePool +Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. To configure Karpenter, you create [NodePools]({{}}) that define how Karpenter manages unschedulable pods and configures nodes. You will also define behaviors for your NodePools, capturing details like how Karpenter handles disruption of nodes and setting limits and weights for each NodePool. Here are some things to know about Karpenter's NodePools: diff --git a/website/content/en/docs/concepts/disruption.md b/website/content/en/docs/concepts/disruption.md index 3dcbf0942779..001de3a242ff 100644 --- a/website/content/en/docs/concepts/disruption.md +++ b/website/content/en/docs/concepts/disruption.md @@ -190,7 +190,7 @@ If you require handling for Spot Rebalance Recommendations, you can use the [AWS Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). -To enable interruption handling, configure the `--interruption-queue-name` CLI argument with the name of the interruption queue provisioned to handle interruption events. +To enable interruption handling, configure the `--interruption-queue` CLI argument with the name of the interruption queue provisioned to handle interruption events. ## Controls @@ -227,7 +227,7 @@ spec: #### Schedule Schedule is a cronjob schedule. Generally, the cron syntax is five space-delimited values with options below, with additional special macros like `@yearly`, `@monthly`, `@weekly`, `@daily`, `@hourly`. -Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#writing-a-cronjob-spec) for more information on how to follow the cron syntax. +Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#writing-a-cronjob-spec) for more information on how to follow the cron syntax. Timezones are not currently supported. Schedules are always in UTC. ```bash # ┌───────────── minute (0 - 59) @@ -241,10 +241,6 @@ Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/worklo # * * * * * ``` -{{% alert title="Note" color="primary" %}} -Timezones are not supported. Most images default to UTC, but it is best practice to ensure this is the case when considering how to define your budgets. -{{% /alert %}} - #### Duration Duration allows compound durations with minutes and hours values such as `10h5m` or `30m` or `160h`. Since cron syntax does not accept denominations smaller than minutes, users can only define minutes or hours. @@ -295,7 +291,7 @@ metadata: #### Example: Disable Disruption on a NodePool -NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in disruption actions. +To disable disruption for all nodes launched by a NodePool, you can configure its `.spec.disruption.budgets`. Setting a budget of zero nodes will prevent any of those nodes from being considered for voluntary disruption. ```yaml apiVersion: karpenter.sh/v1beta1 @@ -303,8 +299,7 @@ kind: NodePool metadata: name: default spec: - template: - metadata: - annotations: # will be applied to all nodes - karpenter.sh/do-not-disrupt: "true" + disruption: + budgets: + - nodes: "0" ``` diff --git a/website/content/en/docs/concepts/nodeclasses.md b/website/content/en/docs/concepts/nodeclasses.md index d426f579e7e6..d9726cea5e44 100644 --- a/website/content/en/docs/concepts/nodeclasses.md +++ b/website/content/en/docs/concepts/nodeclasses.md @@ -160,7 +160,7 @@ status: # Generated instance profile name from "role" instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" ``` -Refer to the [NodePool docs]({{}}) for settings applicable to all providers. To explore various `EC2NodeClass` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). +Refer to the [NodePool docs]({{}}) for settings applicable to all providers. To explore various `EC2NodeClass` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/v0.37.0/examples/v1beta1/). ## spec.amiFamily @@ -431,6 +431,10 @@ spec: AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). **When you specify `amiSelectorTerms`, you fully override the default AMIs that are selected on by your EC2NodeClass [`amiFamily`]({{< ref "#specamifamily" >}}).** +{{% alert title="Note" color="primary" %}} +[`amiFamily`]({{< ref "#specamifamily" >}}) determines the bootstrapping mode, while `amiSelectorTerms` specifies specific AMIs to be used. Therefore, you need to ensure consistency between [`amiFamily`]({{< ref "#specamifamily" >}}) and `amiSelectorTerms` to avoid conflicts during bootstrapping. +{{% /alert %}} + This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. ```yaml @@ -745,7 +749,7 @@ spec: chown -R ec2-user ~ec2-user/.ssh ``` -For more examples on configuring fields for different AMI families, see the [examples here](https://github.com/aws/karpenter/blob/main/examples/v1beta1). +For more examples on configuring fields for different AMI families, see the [examples here](https://github.com/aws/karpenter/blob/v0.37.0/examples/v1beta1/). Karpenter will merge the userData you specify with the default userData for that AMIFamily. See the [AMIFamily]({{< ref "#specamifamily" >}}) section for more details on these defaults. View the sections below to understand the different merge strategies for each AMIFamily. @@ -1242,3 +1246,36 @@ spec: status: instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" ``` + +## status.conditions + +[`status.conditions`]({{< ref "#statusconditions" >}}) indicates EC2NodeClass readiness. This will be `Ready` when Karpenter successfully discovers AMIs, Instance Profile, Subnets, Cluster CIDR and SecurityGroups for the EC2NodeClass. + +```yaml +spec: + role: "KarpenterNodeRole-${CLUSTER_NAME}" +status: + conditions: + Last Transition Time: 2024-05-06T06:04:45Z + Message: Ready + Reason: Ready + Status: True + Type: Ready +``` + +If any of the underlying conditions are not resolved then `Status` is `False` and `Message` indicates the dependency that was not resolved. + +```yaml +spec: + role: "KarpenterNodeRole-${CLUSTER_NAME}" +status: + conditions: + Last Transition Time: 2024-05-06T06:19:46Z + Message: unable to resolve instance profile for node class + Reason: NodeClassNotReady + Status: False + Type: Ready +``` +{{% alert title="Note" color="primary" %}} +An EC2NodeClass that uses AL2023 requires the cluster CIDR for launching nodes. Cluster CIDR will not be resolved for EC2NodeClass that doesn't use AL2023. +{{% /alert %}} diff --git a/website/content/en/docs/concepts/nodepools.md b/website/content/en/docs/concepts/nodepools.md index d679bb5f4373..9d72c848be20 100644 --- a/website/content/en/docs/concepts/nodepools.md +++ b/website/content/en/docs/concepts/nodepools.md @@ -22,7 +22,7 @@ Here are things you should know about NodePools: * If Karpenter encounters a startup taint in the NodePool it will be applied to nodes that are provisioned, but pods do not need to tolerate the taint. Karpenter assumes that the taint is temporary and some other system will remove the taint. * It is recommended to create NodePools that are mutually exclusive. So no Pod should match multiple NodePools. If multiple NodePools are matched, Karpenter will use the NodePool with the highest [weight](#specweight). -For some example `NodePool` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). +For some example `NodePool` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/v0.37.0/examples/v1beta1/). ```yaml apiVersion: karpenter.sh/v1beta1 @@ -157,7 +157,7 @@ spec: duration: 8h nodes: "0" - # Resource limits constrain the total size of the cluster. + # Resource limits constrain the total size of the pool. # Limits prevent Karpenter from creating new instances once the limit is exceeded. limits: cpu: "1000" @@ -233,6 +233,10 @@ Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand i Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. +{{% alert title="Note" color="primary" %}} +There is currently a limit of 30 on the total number of requirements on both the NodePool and the NodeClaim. It's important to note that `spec.template.metadata.labels` are also propagated as requirements on the NodeClaim when it's created, meaning that you can't have more than 30 requirements and labels combined set on your NodePool. +{{% /alert %}} + ### Min Values Along with the combination of [key,operator,values] in the requirements, Karpenter also supports `minValues` in the NodePool requirements block, allowing the scheduler to be aware of user-specified flexibility minimums while scheduling pods to a cluster. If Karpenter cannot meet this minimum flexibility for each key when scheduling a pod, it will fail the scheduling loop for that NodePool, either falling back to another NodePool which meets the pod requirements or failing scheduling the pod altogether. diff --git a/website/content/en/docs/concepts/scheduling.md b/website/content/en/docs/concepts/scheduling.md index 76c09cbc5124..437a6f1cda5f 100755 --- a/website/content/en/docs/concepts/scheduling.md +++ b/website/content/en/docs/concepts/scheduling.md @@ -104,8 +104,6 @@ Refer to general [Kubernetes GPU](https://kubernetes.io/docs/tasks/manage-gpus/s You must enable Pod ENI support in the AWS VPC CNI Plugin before enabling Pod ENI support in Karpenter. Please refer to the [Security Groups for Pods documentation](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) for instructions. {{% /alert %}} -Now that Pod ENI support is enabled in the AWS VPC CNI Plugin, you can enable Pod ENI support in Karpenter by setting the `settings.aws.enablePodENI` Helm chart value to `true`. - Here is an example of a pod-eni resource defined in a deployment manifest: ``` spec: @@ -154,6 +152,7 @@ Take care to ensure the label domains are correct. A well known label like `karp | karpenter.k8s.aws/instance-cpu | 32 | [AWS Specific] Number of CPUs on the instance | | karpenter.k8s.aws/instance-cpu-manufacturer | aws | [AWS Specific] Name of the CPU manufacturer | | karpenter.k8s.aws/instance-memory | 131072 | [AWS Specific] Number of mebibytes of memory on the instance | +| karpenter.k8s.aws/instance-ebs-bandwidth | 9500 | [AWS Specific] Number of [maximum megabits](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-optimized.html#ebs-optimization-performance) of EBS available on the instance | | karpenter.k8s.aws/instance-network-bandwidth | 131072 | [AWS Specific] Number of [baseline megabits](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html) available on the instance | | karpenter.k8s.aws/instance-pods | 110 | [AWS Specific] Number of pods the instance supports | | karpenter.k8s.aws/instance-gpu-name | t4 | [AWS Specific] Name of the GPU on the instance, if available | @@ -175,6 +174,9 @@ requirements: - key: user.defined.label/type operator: Exists ``` +{{% alert title="Note" color="primary" %}} +There is currently a limit of 30 on the total number of requirements on both the NodePool and the NodeClaim. It's important to note that `spec.template.metadata.labels` are also propagated as requirements on the NodeClaim when it's created, meaning that you can't have more than 30 requirements and labels combined set on your NodePool. +{{% /alert %}} #### Node selectors @@ -282,7 +284,7 @@ spec: - p3 taints: - key: nvidia.com/gpu - value: true + value: "true" effect: "NoSchedule" ``` @@ -626,19 +628,73 @@ If using Gt/Lt operators, make sure to use values under the actual label values The `Exists` operator can be used on a NodePool to provide workload segregation across nodes. ```yaml -... -requirements: -- key: company.com/team - operator: Exists +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +spec: + template: + spec: + requirements: + - key: company.com/team + operator: Exists ... ``` -With the requirement on the NodePool, workloads can optionally specify a custom value as a required node affinity or node selector. Karpenter will then label the nodes it launches for these pods which prevents `kube-scheduler` from scheduling conflicting pods to those nodes. This provides a way to more dynamically isolate workloads without requiring a unique NodePool for each workload subset. +With this requirement on the NodePool, workloads can specify the same key (e.g. `company.com/team`) with custom values (e.g. `team-a`, `team-b`, etc.) as a required `nodeAffinity` or `nodeSelector`. Karpenter will then apply the key/value pair to nodes it launches dynamically based on the pod's node requirements. + +If each set of pods that can schedule with this NodePool specifies this key in its `nodeAffinity` or `nodeSelector`, you can isolate pods onto different nodes based on their values. This provides a way to more dynamically isolate workloads without requiring a unique NodePool for each workload subset. + +For example, providing the following `nodeSelectors` would isolate the pods for each of these deployments on different nodes. + +#### Team A Deployment ```yaml -nodeSelector: - company.com/team: team-a +apiVersion: v1 +kind: Deployment +metadata: + name: team-a-deployment +spec: + replicas: 5 + template: + spec: + nodeSelector: + company.com/team: team-a ``` + +#### Team A Node + +```yaml +apiVersion: v1 +kind: Node +metadata: + labels: + company.com/team: team-a +``` + +#### Team B Deployment + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: team-b-deployment +spec: + replicas: 5 + template: + spec: + nodeSelector: + company.com/team: team-b +``` + +#### Team B Node + +```yaml +apiVersion: v1 +kind: Node +metadata: + labels: + company.com/team: team-b +``` + {{% alert title="Note" color="primary" %}} If a workload matches the NodePool but doesn't specify a label, Karpenter will generate a random label for the node. {{% /alert %}} diff --git a/website/content/en/docs/contributing/development-guide.md b/website/content/en/docs/contributing/development-guide.md index 93796441ea98..02632af9222e 100644 --- a/website/content/en/docs/contributing/development-guide.md +++ b/website/content/en/docs/contributing/development-guide.md @@ -84,13 +84,13 @@ By default, `make apply` will set the log level to debug. You can change the log OSX: ```bash -open http://localhost:8000/metrics && kubectl port-forward service/karpenter -n karpenter 8000 +open http://localhost:8080/metrics && kubectl port-forward service/karpenter -n kube-system 8080 ``` Linux: ```bash -gio open http://localhost:8000/metrics && kubectl port-forward service/karpenter -n karpenter 8000 +gio open http://localhost:8080/metrics && kubectl port-forward service/karpenter -n karpenter 8080 ``` ### Tailing Logs @@ -143,8 +143,8 @@ go install github.com/google/pprof@latest ### Get a profile ``` # Connect to the metrics endpoint -kubectl port-forward service/karpenter -n karpenter 8000 -open http://localhost:8000/debug/pprof/ +kubectl port-forward service/karpenter -n karpenter 8080 +open http://localhost:8080/debug/pprof/ # Visualize the memory -go tool pprof -http 0.0.0.0:9000 localhost:8000/debug/pprof/heap +go tool pprof -http 0.0.0.0:9000 localhost:8080/debug/pprof/heap ``` diff --git a/website/content/en/docs/faq.md b/website/content/en/docs/faq.md index c765ef174e2d..80a49837e625 100644 --- a/website/content/en/docs/faq.md +++ b/website/content/en/docs/faq.md @@ -14,10 +14,10 @@ See [Configuring NodePools]({{< ref "./concepts/#configuring-nodepools" >}}) for AWS is the first cloud provider supported by Karpenter, although it is designed to be used with other cloud providers as well. ### Can I write my own cloud provider for Karpenter? -Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.36.0/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. +Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.37.0/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. ### What operating system nodes does Karpenter deploy? -Karpenter uses the OS defined by the [AMI Family in your EC2NodeClass]({{< ref "./concepts/nodeclasses#specamifamily" >}}). +Karpenter uses the OS defined by the [AMI Family in your EC2NodeClass]({{< ref "./concepts/nodeclasses#specamifamily" >}}). ### Can I provide my own custom operating system images? Karpenter has multiple mechanisms for configuring the [operating system]({{< ref "./concepts/nodeclasses/#specamiselectorterms" >}}) for your nodes. @@ -26,7 +26,7 @@ Karpenter has multiple mechanisms for configuring the [operating system]({{< ref Karpenter is flexible to multi-architecture configurations using [well known labels]({{< ref "./concepts/scheduling/#supported-labels">}}). ### What RBAC access is required? -All the required RBAC rules can be found in the Helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.36.0/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.36.0/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.36.0/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.36.0/charts/karpenter/templates/role.yaml) files for details. +All the required RBAC rules can be found in the Helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.37.0/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.37.0/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.37.0/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.37.0/charts/karpenter/templates/role.yaml) files for details. ### Can I run Karpenter outside of a Kubernetes cluster? Yes, as long as the controller has network and IAM/RBAC access to the Kubernetes API and your provider API. @@ -211,15 +211,15 @@ For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgra ### How do I upgrade an EKS Cluster with Karpenter? -When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift is enabled by default starting `0.33.0`. - {{% alert title="Note" color="primary" %}} -Karpenter's default [EC2NodeClass `amiFamily` configuration]({{}}) uses the latest EKS Optimized AL2 AMI for the same major and minor version as the EKS cluster's control plane, meaning that an upgrade of the control plane will cause Karpenter to auto-discover the new AMIs for that version. +Karpenter recommends that you always validate AMIs in your lower environments before using them in production environments. Read [Managing AMIs]({{}}) to understand best practices about upgrading your AMIs. -If using a custom AMI, you will need to trigger the rollout of this new worker node image through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. +If using a custom AMI, you will need to trigger the rollout of new worker node images through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. {{% /alert %}} -Start by [upgrading the EKS Cluster control plane](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). After the EKS Cluster upgrade completes, Karpenter's Drift feature will detect that the Karpenter-provisioned nodes are using EKS Optimized AMIs for the previous cluster version, and [automatically cordon, drain, and replace those nodes]({{}}). To support pods moving to new nodes, follow Kubernetes best practices by setting appropriate pod [Resource Quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/), and using [Pod Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (PDB). Karpenter's Drift feature will spin up replacement nodes based on the pod resource requests, and will respect the PDBs when deprovisioning nodes. +Karpenter's default behavior will upgrade your nodes when you've upgraded your Amazon EKS Cluster. Karpenter will [drift]({{}}) nodes to stay in-sync with the EKS control plane version. Drift is enabled by default starting in `v0.33`. This means that as soon as your cluster is upgraded, Karpenter will auto-discover the new AMIs for that version. + +Start by [upgrading the EKS Cluster control plane](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). After the EKS Cluster upgrade completes, Karpenter will Drift and disrupt the Karpenter-provisioned nodes using EKS Optimized AMIs for the previous cluster version by first spinning up replacement nodes. Karpenter respects [Pod Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (PDB), and automatically [replaces, cordons, and drains those nodes]({{}}). To best support pods moving to new nodes, follow Kubernetes best practices by setting appropriate pod [Resource Quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/) and using PDBs. ## Interruption Handling @@ -231,7 +231,7 @@ Karpenter's native interruption handling offers two main benefits over the stand 1. You don't have to manage and maintain a separate component to exclusively handle interruption events. 2. Karpenter's native interruption handling coordinates with other deprovisioning so that consolidation, expiration, etc. can be aware of interruption events and vice-versa. -### Why am I receiving QueueNotFound errors when I set `--interruption-queue-name`? +### Why am I receiving QueueNotFound errors when I set `--interruption-queue`? Karpenter requires a queue to exist that receives event messages from EC2 and health services in order to handle interruption messages properly for nodes. Details on the types of events that Karpenter handles can be found in the [Interruption Handling Docs]({{< ref "./concepts/disruption/#interruption" >}}). diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/docs/getting-started/getting-started-with-karpenter/_index.md index 89242a872341..26b6b5de8ad8 100644 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/_index.md +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/_index.md @@ -32,7 +32,7 @@ Install these tools before proceeding: 1. [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html) 2. `kubectl` - [the Kubernetes CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) -3. `eksctl` (>= v0.169.0) - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) +3. `eksctl` (>= v0.180.0) - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) 4. `helm` - [the package manager for Kubernetes](https://helm.sh/docs/intro/install/) [Configure the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) @@ -45,8 +45,8 @@ After setting up the tools, set the Karpenter and Kubernetes version: ```bash export KARPENTER_NAMESPACE="kube-system" -export KARPENTER_VERSION="0.36.0" -export K8S_VERSION="1.29" +export KARPENTER_VERSION="0.37.0" +export K8S_VERSION="1.30" ``` Then set the following environment variable: @@ -87,6 +87,9 @@ The following cluster configuration will: {{% /tab %}} {{< /tabpane >}} +Unless your AWS account has already onboarded to EC2 Spot, you will need to create the service linked role to +avoid the [`ServiceLinkedRoleCreationNotPermitted` error]({{}}). + {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh" language="bash"%}} {{% alert title="Windows Support Notice" color="warning" %}} @@ -109,13 +112,13 @@ See [Enabling Windows support](https://docs.aws.amazon.com/eks/latest/userguide/ As the OCI Helm chart is signed by [Cosign](https://github.com/sigstore/cosign) as part of the release process you can verify the chart before installing it by running the following command. ```bash -cosign verify public.ecr.aws/karpenter/karpenter:0.36.0 \ +cosign verify public.ecr.aws/karpenter/karpenter:0.37.0 \ --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ --certificate-identity-regexp='https://github\.com/aws/karpenter-provider-aws/\.github/workflows/release\.yaml@.+' \ --certificate-github-workflow-repository=aws/karpenter-provider-aws \ --certificate-github-workflow-name=Release \ - --certificate-github-workflow-ref=refs/tags/v0.36.0 \ - --annotations version=0.36.0 + --certificate-github-workflow-ref=refs/tags/v0.37.0 \ + --annotations version=0.37.0 ``` {{% alert title="DNS Policy Notice" color="warning" %}} diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml b/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml index f80755267455..1878cd6d352a 100644 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml @@ -213,7 +213,7 @@ Resources: { "Sid": "AllowScopedInstanceProfileCreationActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:CreateInstanceProfile" ], @@ -230,7 +230,7 @@ Resources: { "Sid": "AllowScopedInstanceProfileTagActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:TagInstanceProfile" ], @@ -250,7 +250,7 @@ Resources: { "Sid": "AllowScopedInstanceProfileActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:AddRoleToInstanceProfile", "iam:RemoveRoleFromInstanceProfile", @@ -269,7 +269,7 @@ Resources: { "Sid": "AllowInstanceProfileReadActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": "iam:GetInstanceProfile" }, { @@ -301,6 +301,14 @@ Resources: - sqs.amazonaws.com Action: sqs:SendMessage Resource: !GetAtt KarpenterInterruptionQueue.Arn + - Sid: DenyHTTP + Effect: Deny + Action: sqs:* + Resource: !GetAtt KarpenterInterruptionQueue.Arn + Condition: + Bool: + aws:SecureTransport: false + Principal: "*" ScheduledChangeRule: Type: 'AWS::Events::Rule' Properties: @@ -344,4 +352,4 @@ Resources: - EC2 Instance State-change Notification Targets: - Id: KarpenterInterruptionQueueTarget - Arn: !GetAtt KarpenterInterruptionQueue.Arn \ No newline at end of file + Arn: !GetAtt KarpenterInterruptionQueue.Arn diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/grafana-values.yaml b/website/content/en/docs/getting-started/getting-started-with-karpenter/grafana-values.yaml index 15a60c94f47d..5f239b186f25 100644 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/grafana-values.yaml +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/grafana-values.yaml @@ -22,6 +22,6 @@ dashboardProviders: dashboards: default: capacity-dashboard: - url: https://karpenter.sh/v0.36/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json + url: https://karpenter.sh/v0.37/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json performance-dashboard: - url: https://karpenter.sh/v0.36/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json + url: https://karpenter.sh/v0.37/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json b/website/content/en/docs/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json index d474d01f4e16..7f93053b3206 100644 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json @@ -1,1440 +1,1698 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 2, - "id": 6, - "links": [], - "liveNow": true, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(action, cluster) (karpenter_deprovisioning_actions_performed)", - "format": "time_series", - "instant": false, - "legendFormat": "{{cluster}}: {{action}}", - "range": true, - "refId": "A" - } - ], - "title": "Deprovisioning Actions Performed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "sum by(cluster) (karpenter_nodes_created)", - "format": "time_series", - "legendFormat": "{{cluster}}", - "range": true, - "refId": "A" - } - ], - "title": "Nodes Created", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 15, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "sum by(cluster) (karpenter_nodes_terminated)", - "format": "time_series", - "legendFormat": "{{cluster}}", - "range": true, - "refId": "A" - } - ], - "title": "Nodes Terminated", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by(phase)(karpenter_pods_state)", - "legendFormat": "{{label_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Pod Phase", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", - "legendFormat": "{{label_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Pod Distribution: $distribution_filter", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-RdYlGr" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*Utilization$" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 1 - }, - { - "id": "unit", - "value": "percentunit" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Provisioned" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 18, - "x": 0, - "y": 29 - }, - "id": 10, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "9.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "karpenter_nodepool_usage{resource_type=\"cpu\"} / karpenter_nodepool_limit{resource_type=\"cpu\"}", - "format": "table", - "instant": true, - "legendFormat": "CPU Limit Utilization", - "range": false, - "refId": "CPU Limit Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Node Count" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "karpenter_nodepool_usage{resource_type=\"memory\"} / karpenter_nodepool_limit{resource_type=\"memory\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Memory Limit Utilization", - "range": false, - "refId": "Memory Limit Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "CPU Capacity" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Memory Capacity" - } - ], - "title": "Nodepool Summary", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "nodepool" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "__name__": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "job": true, - "job 1": true, - "job 2": true, - "resource_type": true, - "resource_type 1": true, - "resource_type 2": true - }, - "indexByName": { - "Time 1": 6, - "Time 2": 7, - "Time 3": 11, - "Time 4": 15, - "Time 5": 16, - "Value #CPU Capacity": 2, - "Value #CPU Limit Utilization": 3, - "Value #Memory Capacity": 4, - "Value #Memory Limit Utilization": 5, - "Value #Node Count": 1, - "instance 1": 8, - "instance 2": 12, - "job 1": 9, - "job 2": 13, - "nodepool": 0, - "resource_type 1": 10, - "resource_type 2": 14 - }, - "renameByName": { - "Time 1": "", - "Value": "CPU Utilization", - "Value #CPU Capacity": "CPU Provisioned", - "Value #CPU Limit Utilization": "CPU Limit Utilization", - "Value #CPU Utilization": "CPU Limit Utilization", - "Value #Memory Capacity": "Memory Provisioned", - "Value #Memory Limit Utilization": "Memory Limit Utilization", - "Value #Memory Utilization": "Memory Utilization", - "Value #Node Count": "Node Count", - "instance": "", - "instance 1": "", - "job": "", - "nodepool": "Nodepool" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 18, - "y": 29 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", - "legendFormat": "Percentage", - "range": true, - "refId": "A" - } - ], - "title": "Spot Node Percentage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-RdYlGr" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "node_name" - }, - "properties": [ - { - "id": "custom.width", - "value": 333 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Utilization" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "thresholds", - "value": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 75 - } - ] - } - }, - { - "id": "max", - "value": 1 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "unit", - "value": "s" - }, - { - "id": "decimals", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 4, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Uptime" - } - ] - }, - "pluginVersion": "9.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "CPU Utilization", - "range": false, - "refId": "CPU Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Memory Utilization", - "range": false, - "refId": "Memory Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Memory Utilization", - "range": false, - "refId": "Pod Count" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\n sum by (node)(node_time_seconds) - sum by (node)(node_boot_time_seconds),\n \"node_name\", \"$1\", \"node\", \"(.+)\"\n)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Uptime", - "range": false, - "refId": "Uptime" - } - ], - "title": "Node Summary", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "node_name" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Value": false, - "Value #Pod Count": false, - "__name__": true, - "arch": true, - "arch 1": true, - "arch 2": true, - "arch 3": true, - "capacity_type 2": true, - "capacity_type 3": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "instance 3": true, - "instance_category 1": true, - "instance_category 2": true, - "instance_category 3": true, - "instance_cpu": true, - "instance_cpu 1": true, - "instance_cpu 2": true, - "instance_cpu 3": true, - "instance_family": true, - "instance_family 1": true, - "instance_family 2": true, - "instance_family 3": true, - "instance_generation 1": true, - "instance_generation 2": true, - "instance_generation 3": true, - "instance_gpu_count": true, - "instance_gpu_count 1": true, - "instance_gpu_count 2": true, - "instance_gpu_count 3": true, - "instance_gpu_manufacturer": true, - "instance_gpu_manufacturer 1": true, - "instance_gpu_manufacturer 2": true, - "instance_gpu_manufacturer 3": true, - "instance_gpu_memory": true, - "instance_gpu_memory 1": true, - "instance_gpu_memory 2": true, - "instance_gpu_memory 3": true, - "instance_gpu_name": true, - "instance_gpu_name 1": true, - "instance_gpu_name 2": true, - "instance_gpu_name 3": true, - "instance_hypervisor": true, - "instance_hypervisor 1": true, - "instance_hypervisor 2": true, - "instance_hypervisor 3": true, - "instance_local_nvme 1": true, - "instance_local_nvme 2": true, - "instance_local_nvme 3": true, - "instance_memory": true, - "instance_memory 1": true, - "instance_memory 2": true, - "instance_memory 3": true, - "instance_pods": true, - "instance_pods 1": true, - "instance_pods 2": true, - "instance_pods 3": true, - "instance_size": true, - "instance_size 1": true, - "instance_size 2": true, - "instance_size 3": true, - "instance_type 1": false, - "instance_type 2": true, - "instance_type 3": true, - "job": true, - "job 1": true, - "job 2": true, - "job 3": true, - "node": true, - "os": true, - "os 1": true, - "os 2": true, - "os 3": true, - "nodepool 1": false, - "nodepool 2": true, - "nodepool 3": true, - "resource_type": true, - "resource_type 1": true, - "resource_type 2": true, - "resource_type 3": true, - "zone 1": false, - "zone 2": true, - "zone 3": true - }, - "indexByName": { - "Time 1": 1, - "Time 2": 25, - "Time 3": 45, - "Time 4": 65, - "Value #CPU Utilization": 10, - "Value #Memory Utilization": 11, - "Value #Pod Count": 9, - "Value #Uptime": 8, - "arch 1": 5, - "arch 2": 26, - "arch 3": 46, - "capacity_type 1": 6, - "capacity_type 2": 27, - "capacity_type 3": 47, - "instance 1": 4, - "instance 2": 28, - "instance 3": 48, - "instance_cpu 1": 12, - "instance_cpu 2": 29, - "instance_cpu 3": 49, - "instance_family 1": 13, - "instance_family 2": 30, - "instance_family 3": 50, - "instance_gpu_count 1": 14, - "instance_gpu_count 2": 31, - "instance_gpu_count 3": 51, - "instance_gpu_manufacturer 1": 15, - "instance_gpu_manufacturer 2": 32, - "instance_gpu_manufacturer 3": 52, - "instance_gpu_memory 1": 16, - "instance_gpu_memory 2": 33, - "instance_gpu_memory 3": 53, - "instance_gpu_name 1": 17, - "instance_gpu_name 2": 34, - "instance_gpu_name 3": 54, - "instance_hypervisor 1": 18, - "instance_hypervisor 2": 35, - "instance_hypervisor 3": 55, - "instance_memory 1": 19, - "instance_memory 2": 36, - "instance_memory 3": 56, - "instance_pods 1": 20, - "instance_pods 2": 37, - "instance_pods 3": 57, - "instance_size 1": 21, - "instance_size 2": 38, - "instance_size 3": 58, - "instance_type 1": 3, - "instance_type 2": 39, - "instance_type 3": 59, - "job 1": 22, - "job 2": 40, - "job 3": 60, - "node": 66, - "node_name": 0, - "os 1": 23, - "os 2": 41, - "os 3": 61, - "nodepool 1": 2, - "nodepool 2": 42, - "nodepool 3": 62, - "resource_type 1": 24, - "resource_type 2": 43, - "resource_type 3": 63, - "zone 1": 7, - "zone 2": 44, - "zone 3": 64 - }, - "renameByName": { - "Time": "", - "Time 1": "", - "Value": "CPU Utilization", - "Value #Allocatable": "", - "Value #CPU Utilization": "CPU Utilization", - "Value #Memory Utilization": "Memory Utilization", - "Value #Pod CPU": "", - "Value #Pod Count": "Pods", - "Value #Uptime": "Uptime", - "arch": "Architecture", - "arch 1": "Arch", - "capacity_type": "Capacity Type", - "capacity_type 1": "Capacity Type", - "instance 1": "Instance", - "instance_cpu 1": "vCPU", - "instance_type": "Instance Type", - "instance_type 1": "Instance Type", - "node_name": "Node Name", - "nodepool 1": "Nodepool", - "zone 1": "Zone" - } - } - } - ], - "type": "table" - } - ], - "refresh": false, - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "includeAll": false, - "label": "Data Source", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, nodepool)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "nodepool", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, nodepool)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, zone)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "zone", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, zone)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, arch)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "arch", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, arch)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, capacity_type)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "capacity_type", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, capacity_type)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, instance_type)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "instance_type", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, instance_type)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": "nodepool", - "value": "nodepool" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "distribution_filter", - "options": [ - { - "selected": false, - "text": "arch", - "value": "arch" - }, - { - "selected": false, - "text": "capacity_type", - "value": "capacity_type" - }, - { - "selected": false, - "text": "instance_type", - "value": "instance_type" - }, - { - "selected": false, - "text": "namespace", - "value": "namespace" - }, - { - "selected": false, - "text": "node", - "value": "node" - }, - { - "selected": true, - "text": "nodepool", - "value": "nodepool" - }, - { - "selected": false, - "text": "zone", - "value": "zone" - } - ], - "query": "arch,capacity_type,instance_type,namespace,node,nodepool,zone", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Karpenter Capacity", - "uid": "ta8I9Q67z", - "version": 4, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 2, + "id": 44, + "links": [], + "liveNow": true, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(cluster,nodepool) (karpenter_nodes_created{nodepool=~\"$nodepool\"})", + "format": "time_series", + "legendFormat": "{{cluster}}", + "range": true, + "refId": "A" + } + ], + "title": "Nodes Created: nodepool \"$nodepool\"", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(cluster,nodepool) (karpenter_nodes_terminated{nodepool=~\"$nodepool\"})", + "format": "time_series", + "legendFormat": "{{cluster}}", + "range": true, + "refId": "A" + } + ], + "title": "Nodes Terminated: nodepool \"$nodepool\"", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(consolidation_type,method)(karpenter_disruption_eligible_nodes)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Nodes Eligible for Disruptions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(action,consolidation_type,method)(karpenter_disruption_actions_performed_total)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Disruption Actions Performed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "See: https://karpenter.sh/v0.35/concepts/disruption/#automated-methods", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(action,consolidation_type,method)(karpenter_disruption_nodes_disrupted_total{nodepool=~\"$nodepool\"})", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Voluntary Node Disruptions: nodepool \"$nodepool\"", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pod Distribution: $distribution_filter", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(phase)(karpenter_pods_state)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pod Phase", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*Utilization$" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 1 + }, + { + "id": "unit", + "value": "percentunit" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Provisioned" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 18, + "x": 0, + "y": 42 + }, + "id": 10, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "karpenter_nodepool_usage{resource_type=\"cpu\"} / karpenter_nodepool_limit{resource_type=\"cpu\"}", + "format": "table", + "instant": true, + "legendFormat": "CPU Limit Utilization", + "range": false, + "refId": "CPU Limit Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "Node Count" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "karpenter_nodepool_usage{resource_type=\"memory\"} / karpenter_nodepool_limit{resource_type=\"memory\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Memory Limit Utilization", + "range": false, + "refId": "Memory Limit Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "CPU Capacity" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "Memory Capacity" + } + ], + "title": "Nodepool Summary", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "nodepool" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "__name__": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "job": true, + "job 1": true, + "job 2": true, + "resource_type": true, + "resource_type 1": true, + "resource_type 2": true + }, + "indexByName": { + "Time 1": 6, + "Time 2": 7, + "Time 3": 11, + "Time 4": 15, + "Time 5": 16, + "Value #CPU Capacity": 2, + "Value #CPU Limit Utilization": 3, + "Value #Memory Capacity": 4, + "Value #Memory Limit Utilization": 5, + "Value #Node Count": 1, + "instance 1": 8, + "instance 2": 12, + "job 1": 9, + "job 2": 13, + "nodepool": 0, + "resource_type 1": 10, + "resource_type 2": 14 + }, + "renameByName": { + "Time 1": "", + "Value": "CPU Utilization", + "Value #CPU Capacity": "CPU Provisioned", + "Value #CPU Limit Utilization": "CPU Limit Utilization", + "Value #CPU Utilization": "CPU Limit Utilization", + "Value #Memory Capacity": "Memory Provisioned", + "Value #Memory Limit Utilization": "Memory Limit Utilization", + "Value #Memory Utilization": "Memory Utilization", + "Value #Node Count": "Node Count", + "instance": "", + "instance 1": "", + "job": "", + "nodepool": "Nodepool" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 18, + "y": 42 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", + "legendFormat": "Percentage", + "range": true, + "refId": "A" + } + ], + "title": "Spot Node Percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "node_name" + }, + "properties": [ + { + "id": "custom.width", + "value": 333 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Utilization" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "percentage", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 75 + } + ] + } + }, + { + "id": "max", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Uptime" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "decimals", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Uptime" + } + ] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "CPU Utilization", + "range": false, + "refId": "CPU Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Memory Utilization", + "range": false, + "refId": "Memory Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Memory Utilization", + "range": false, + "refId": "Pod Count" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\n sum by (node)(node_time_seconds) - sum by (node)(node_boot_time_seconds),\n \"node_name\", \"$1\", \"node\", \"(.+)\"\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Uptime", + "range": false, + "refId": "Uptime" + } + ], + "title": "Node Summary", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "node_name" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Value": false, + "Value #Pod Count": false, + "__name__": true, + "arch": true, + "arch 1": true, + "arch 2": true, + "arch 3": true, + "capacity_type 2": true, + "capacity_type 3": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "instance 3": true, + "instance_category 1": true, + "instance_category 2": true, + "instance_category 3": true, + "instance_cpu": true, + "instance_cpu 1": true, + "instance_cpu 2": true, + "instance_cpu 3": true, + "instance_family": true, + "instance_family 1": true, + "instance_family 2": true, + "instance_family 3": true, + "instance_generation 1": true, + "instance_generation 2": true, + "instance_generation 3": true, + "instance_gpu_count": true, + "instance_gpu_count 1": true, + "instance_gpu_count 2": true, + "instance_gpu_count 3": true, + "instance_gpu_manufacturer": true, + "instance_gpu_manufacturer 1": true, + "instance_gpu_manufacturer 2": true, + "instance_gpu_manufacturer 3": true, + "instance_gpu_memory": true, + "instance_gpu_memory 1": true, + "instance_gpu_memory 2": true, + "instance_gpu_memory 3": true, + "instance_gpu_name": true, + "instance_gpu_name 1": true, + "instance_gpu_name 2": true, + "instance_gpu_name 3": true, + "instance_hypervisor": true, + "instance_hypervisor 1": true, + "instance_hypervisor 2": true, + "instance_hypervisor 3": true, + "instance_local_nvme 1": true, + "instance_local_nvme 2": true, + "instance_local_nvme 3": true, + "instance_memory": true, + "instance_memory 1": true, + "instance_memory 2": true, + "instance_memory 3": true, + "instance_pods": true, + "instance_pods 1": true, + "instance_pods 2": true, + "instance_pods 3": true, + "instance_size": true, + "instance_size 1": true, + "instance_size 2": true, + "instance_size 3": true, + "instance_type 1": false, + "instance_type 2": true, + "instance_type 3": true, + "job": true, + "job 1": true, + "job 2": true, + "job 3": true, + "node": true, + "nodepool 1": false, + "nodepool 2": true, + "nodepool 3": true, + "os": true, + "os 1": true, + "os 2": true, + "os 3": true, + "resource_type": true, + "resource_type 1": true, + "resource_type 2": true, + "resource_type 3": true, + "zone 1": false, + "zone 2": true, + "zone 3": true + }, + "indexByName": { + "Time 1": 1, + "Time 2": 25, + "Time 3": 45, + "Time 4": 65, + "Value #CPU Utilization": 10, + "Value #Memory Utilization": 11, + "Value #Pod Count": 9, + "Value #Uptime": 8, + "arch 1": 5, + "arch 2": 26, + "arch 3": 46, + "capacity_type 1": 6, + "capacity_type 2": 27, + "capacity_type 3": 47, + "instance 1": 4, + "instance 2": 28, + "instance 3": 48, + "instance_cpu 1": 12, + "instance_cpu 2": 29, + "instance_cpu 3": 49, + "instance_family 1": 13, + "instance_family 2": 30, + "instance_family 3": 50, + "instance_gpu_count 1": 14, + "instance_gpu_count 2": 31, + "instance_gpu_count 3": 51, + "instance_gpu_manufacturer 1": 15, + "instance_gpu_manufacturer 2": 32, + "instance_gpu_manufacturer 3": 52, + "instance_gpu_memory 1": 16, + "instance_gpu_memory 2": 33, + "instance_gpu_memory 3": 53, + "instance_gpu_name 1": 17, + "instance_gpu_name 2": 34, + "instance_gpu_name 3": 54, + "instance_hypervisor 1": 18, + "instance_hypervisor 2": 35, + "instance_hypervisor 3": 55, + "instance_memory 1": 19, + "instance_memory 2": 36, + "instance_memory 3": 56, + "instance_pods 1": 20, + "instance_pods 2": 37, + "instance_pods 3": 57, + "instance_size 1": 21, + "instance_size 2": 38, + "instance_size 3": 58, + "instance_type 1": 3, + "instance_type 2": 39, + "instance_type 3": 59, + "job 1": 22, + "job 2": 40, + "job 3": 60, + "node": 66, + "node_name": 0, + "nodepool 1": 2, + "nodepool 2": 42, + "nodepool 3": 62, + "os 1": 23, + "os 2": 41, + "os 3": 61, + "resource_type 1": 24, + "resource_type 2": 43, + "resource_type 3": 63, + "zone 1": 7, + "zone 2": 44, + "zone 3": 64 + }, + "renameByName": { + "Time": "", + "Time 1": "", + "Value": "CPU Utilization", + "Value #Allocatable": "", + "Value #CPU Utilization": "CPU Utilization", + "Value #Memory Utilization": "Memory Utilization", + "Value #Pod CPU": "", + "Value #Pod Count": "Pods", + "Value #Uptime": "Uptime", + "arch": "Architecture", + "arch 1": "Arch", + "capacity_type": "Capacity Type", + "capacity_type 1": "Capacity Type", + "instance 1": "Instance", + "instance_cpu 1": "vCPU", + "instance_type": "Instance Type", + "instance_type 1": "Instance Type", + "node_name": "Node Name", + "nodepool 1": "Nodepool", + "zone 1": "Zone" + } + } + } + ], + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, nodepool)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "nodepool", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, nodepool)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, zone)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "zone", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, zone)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, arch)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "arch", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, arch)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, capacity_type)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "capacity_type", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, capacity_type)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, instance_type)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "instance_type", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, instance_type)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(karpenter_disruption_actions_performed_total,method)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(karpenter_disruption_actions_performed_total,method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "nodepool", + "value": "nodepool" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "distribution_filter", + "options": [ + { + "selected": false, + "text": "arch", + "value": "arch" + }, + { + "selected": false, + "text": "capacity_type", + "value": "capacity_type" + }, + { + "selected": false, + "text": "instance_type", + "value": "instance_type" + }, + { + "selected": false, + "text": "method", + "value": "method" + }, + { + "selected": false, + "text": "namespace", + "value": "namespace" + }, + { + "selected": false, + "text": "node", + "value": "node" + }, + { + "selected": true, + "text": "nodepool", + "value": "nodepool" + }, + { + "selected": false, + "text": "zone", + "value": "zone" + } + ], + "query": "arch,capacity_type,instance_type,method,namespace,node,nodepool,zone", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Karpenter Capacity", + "uid": "ta8I9Q67z", + "version": 5, + "weekStart": "" } diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json b/website/content/en/docs/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json index c7762d302f99..174282ee9468 100644 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json @@ -498,7 +498,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(controller_runtime_reconcile_total[10m])) by (controller)", + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"karpenter\"}[10m])) by (controller)", "legendFormat": "{{controller}}", "range": true, "refId": "A" @@ -542,14 +542,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(controller_runtime_reconcile_time_seconds_count, controller)", + "definition": "label_values(controller_runtime_reconcile_time_seconds_count{job=\"karpenter\"}, controller)", "hide": 0, "includeAll": false, "multi": false, "name": "controller", "options": [], "query": { - "query": "label_values(controller_runtime_reconcile_time_seconds_count, controller)", + "query": "label_values(controller_runtime_reconcile_time_seconds_count{job=\"karpenter\"}, controller)", "refId": "StandardVariableQuery" }, "refresh": 2, diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh index c17584cec718..0854100d516c 100755 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh @@ -14,12 +14,18 @@ spec: app: inflate spec: terminationGracePeriodSeconds: 0 + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 containers: - - name: inflate - image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 - resources: - requests: - cpu: 1 + - name: inflate + image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 + resources: + requests: + cpu: 1 + securityContext: + allowPrivilegeEscalation: false EOF kubectl scale deployment inflate --replicas 5 diff --git a/website/content/en/docs/getting-started/migrating-from-cas/_index.md b/website/content/en/docs/getting-started/migrating-from-cas/_index.md index f71116389411..dd593240c3b7 100644 --- a/website/content/en/docs/getting-started/migrating-from-cas/_index.md +++ b/website/content/en/docs/getting-started/migrating-from-cas/_index.md @@ -92,7 +92,7 @@ One for your Karpenter node role and one for your existing node group. First set the Karpenter release you want to deploy. ```bash -export KARPENTER_VERSION="0.36.0" +export KARPENTER_VERSION="0.37.0" ``` We can now generate a full Karpenter deployment yaml from the Helm chart. @@ -117,7 +117,6 @@ affinity: - matchExpressions: - key: karpenter.sh/nodepool operator: DoesNotExist - - matchExpressions: - key: eks.amazonaws.com/nodegroup operator: In values: @@ -133,7 +132,7 @@ Now that our deployment is ready we can create the karpenter namespace, create t ## Create default NodePool -We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.36.0/examples/v1beta1) for specific needs. +We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.37.0/examples/v1beta1) for specific needs. {{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh" language="bash" %}} diff --git a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh index 47df188dc87d..31e8093cd6f8 100644 --- a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh +++ b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh @@ -1,6 +1,6 @@ for NODEGROUP in $(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" --query 'nodegroups' --output text); do aws ec2 create-tags \ --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ - --resources "$(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" \ - --nodegroup-name "${NODEGROUP}" --query 'nodegroup.subnets' --output text )" + --resources $(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" \ + --nodegroup-name "${NODEGROUP}" --query 'nodegroup.subnets' --output text ) done diff --git a/website/content/en/docs/reference/cloudformation.md b/website/content/en/docs/reference/cloudformation.md index 1e3993b0b1bf..76b938c73eb4 100644 --- a/website/content/en/docs/reference/cloudformation.md +++ b/website/content/en/docs/reference/cloudformation.md @@ -17,7 +17,7 @@ These descriptions should allow you to understand: To download a particular version of `cloudformation.yaml`, set the version and use `curl` to pull the file to your local system: ```bash -export KARPENTER_VERSION="0.36.0" +export KARPENTER_VERSION="0.37.0" curl https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > cloudformation.yaml ``` @@ -376,14 +376,14 @@ This gives EC2 permission explicit permission to use the `KarpenterNodeRole-${Cl #### AllowScopedInstanceProfileCreationActions The AllowScopedInstanceProfileCreationActions Sid gives the Karpenter controller permission to create a new instance profile with [`iam:CreateInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_CreateInstanceProfile.html), -provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName` set to owned and is made in the current region. +provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName}` set to owned and is made in the current region. Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures that Karpenter can generate instance profiles on your behalf based on roles specified in your `EC2NodeClasses` that you use to configure Karpenter. ```json { "Sid": "AllowScopedInstanceProfileCreationActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:CreateInstanceProfile" ], @@ -408,7 +408,7 @@ Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures t { "Sid": "AllowScopedInstanceProfileTagActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:TagInstanceProfile" ], @@ -431,14 +431,14 @@ Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures t #### AllowScopedInstanceProfileActions The AllowScopedInstanceProfileActions Sid gives the Karpenter controller permission to perform [`iam:AddRoleToInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_AddRoleToInstanceProfile.html), [`iam:RemoveRoleFromInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_RemoveRoleFromInstanceProfile.html), and [`iam:DeleteInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_DeleteInstanceProfile.html) actions, -provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName` set to owned and is made in the current region. +provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName}` set to owned and is made in the current region. Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This permission is further enforced by the `iam:PassRole` permission. If Karpenter attempts to add a role to an instance profile that it doesn't have `iam:PassRole` permission on, that call will fail. Therefore, if you configure Karpenter to use a new role through the `EC2NodeClass`, ensure that you also specify that role within your `iam:PassRole` permission. ```json { "Sid": "AllowScopedInstanceProfileActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:AddRoleToInstanceProfile", "iam:RemoveRoleFromInstanceProfile", @@ -464,7 +464,7 @@ The AllowInstanceProfileActions Sid gives the Karpenter controller permission to { "Sid": "AllowInstanceProfileReadActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": "iam:GetInstanceProfile" } ``` @@ -525,6 +525,7 @@ KarpenterInterruptionQueue: The Karpenter interruption queue policy is created to allow AWS services that we want to receive instance notifications from to push notification messages to the queue. The [AWS::SQS::QueuePolicy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sqs-queuepolicy.html) resource here applies `EC2InterruptionPolicy` to the `KarpenterInterruptionQueue`. The policy allows [sqs:SendMessage](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html) actions to `events.amazonaws.com` and `sqs.amazonaws.com` services. It also allows the `GetAtt` function to get attributes from `KarpenterInterruptionQueue.Arn`. +Additionally, it only allows access to the queue using encrypted connections over HTTPS (TLS) to adhere to [Amazon SQS Security Best Practices](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-security-best-practices.html#enforce-encryption-data-in-transit). ```yaml KarpenterInterruptionQueuePolicy: @@ -542,6 +543,14 @@ KarpenterInterruptionQueuePolicy: - sqs.amazonaws.com Action: sqs:SendMessage Resource: !GetAtt KarpenterInterruptionQueue.Arn + - Sid: DenyHTTP + Effect: Deny + Action: sqs:* + Resource: !GetAtt KarpenterInterruptionQueue.Arn + Condition: + Bool: + aws:SecureTransport: false + Principal: "*" ``` ### Rules diff --git a/website/content/en/docs/reference/instance-types.md b/website/content/en/docs/reference/instance-types.md index b304534088bf..c01e769d9130 100644 --- a/website/content/en/docs/reference/instance-types.md +++ b/website/content/en/docs/reference/instance-types.md @@ -10,8 +10,7 @@ description: > AWS instance types offer varying resources and can be selected by labels. The values provided below are the resources available with some assumptions and after the instance overhead has been subtracted: - `blockDeviceMappings` are not configured -- `aws-eni-limited-pod-density` is assumed to be `true` -- `amiFamily` is set to the default of `AL2` +- `amiFamily` is set to `AL2023` ## a1 Family ### `a1.medium` #### Labels @@ -20,6 +19,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -45,6 +45,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -70,6 +71,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -95,6 +97,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -120,6 +123,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -145,6 +149,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -335,6 +340,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -358,6 +364,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -381,6 +388,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -404,6 +412,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -427,6 +436,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|36| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -452,6 +462,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -477,6 +488,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -502,6 +514,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -527,6 +540,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -552,6 +566,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|36| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -577,6 +592,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -602,6 +618,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|72| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -627,6 +644,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -652,6 +670,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -678,6 +697,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -703,6 +723,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -728,6 +749,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -753,6 +775,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -778,6 +801,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -803,6 +827,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -828,6 +853,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6300| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -853,6 +879,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -879,6 +906,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -905,6 +933,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -931,6 +960,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -957,6 +987,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -983,6 +1014,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -1009,6 +1041,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -1035,6 +1068,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6300| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -1061,6 +1095,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -1088,6 +1123,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1114,6 +1150,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1140,6 +1177,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1166,6 +1204,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1192,6 +1231,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|36| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1218,6 +1258,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1244,6 +1285,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|72| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1270,6 +1312,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1296,6 +1339,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1323,6 +1367,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1348,6 +1393,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1373,6 +1419,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1398,6 +1445,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1423,6 +1471,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|36| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1449,6 +1498,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|72| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1475,6 +1525,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|72| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1502,6 +1553,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1527,6 +1579,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1552,6 +1605,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1577,6 +1631,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1602,6 +1657,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1627,6 +1683,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1652,6 +1709,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1677,6 +1735,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1702,6 +1761,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1727,6 +1787,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1753,6 +1814,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1780,6 +1842,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1805,6 +1868,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1830,6 +1894,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1855,6 +1920,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1880,6 +1946,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1905,6 +1972,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1930,6 +1998,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1955,6 +2024,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1980,6 +2050,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -2006,6 +2077,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2032,6 +2104,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2058,6 +2131,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2084,6 +2158,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2110,6 +2185,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2136,6 +2212,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2162,6 +2239,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2188,6 +2266,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2214,6 +2293,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2241,6 +2321,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2266,6 +2347,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2291,6 +2373,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2316,6 +2399,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2341,6 +2425,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2366,6 +2451,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2391,6 +2477,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|28500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2416,6 +2503,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2443,6 +2531,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2468,6 +2557,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2493,6 +2583,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2518,6 +2609,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2543,6 +2635,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2568,6 +2661,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2593,6 +2687,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2618,6 +2713,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2643,6 +2739,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2669,6 +2766,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2696,6 +2794,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2722,6 +2821,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2748,6 +2848,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2774,6 +2875,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2800,6 +2902,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2826,6 +2929,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2852,6 +2956,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2878,6 +2983,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2904,6 +3010,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2931,6 +3038,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2959,6 +3067,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -2984,6 +3093,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3009,6 +3119,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3034,6 +3145,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3059,6 +3171,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3084,6 +3197,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|37500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3109,6 +3223,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|50000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3134,6 +3249,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|75000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3159,6 +3275,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3174,10 +3291,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|238333Mi| - |pods|345| + |memory|237794Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ### `c6in.metal` #### Labels | Label | Value | @@ -3185,6 +3302,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3200,10 +3318,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|238333Mi| - |pods|345| + |memory|237794Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ## c7a Family ### `c7a.medium` #### Labels @@ -3212,6 +3330,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3237,6 +3356,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3262,6 +3382,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3287,6 +3408,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3312,6 +3434,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3337,6 +3460,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3362,6 +3486,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3387,6 +3512,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3412,6 +3538,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3437,6 +3564,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3462,6 +3590,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3488,6 +3617,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3515,6 +3645,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3540,6 +3671,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3565,6 +3697,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3590,6 +3723,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3615,6 +3749,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3640,6 +3775,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3665,6 +3801,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3690,6 +3827,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3716,6 +3854,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3743,6 +3882,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3769,6 +3909,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3795,6 +3936,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3821,6 +3963,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3847,6 +3990,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3873,6 +4017,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3899,6 +4044,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3925,6 +4071,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3952,6 +4099,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3971,6 +4119,7 @@ below are the resources available with some assumptions and after the instance o |memory|112720Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## c7gn Family ### `c7gn.medium` #### Labels @@ -3979,6 +4128,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4004,6 +4154,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4029,6 +4180,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4054,6 +4206,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4079,6 +4232,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4104,6 +4258,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4129,6 +4284,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4154,6 +4310,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4180,11 +4337,13 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|200000| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -4197,6 +4356,7 @@ below are the resources available with some assumptions and after the instance o |memory|112720Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## c7i Family ### `c7i.large` #### Labels @@ -4205,6 +4365,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4230,6 +4391,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4255,6 +4417,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4280,6 +4443,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4305,6 +4469,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4330,6 +4495,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4355,6 +4521,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4380,6 +4547,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4405,6 +4573,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4430,6 +4599,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4456,6 +4626,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4483,6 +4654,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|d2| |karpenter.k8s.aws/instance-generation|2| @@ -4506,6 +4678,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|d2| |karpenter.k8s.aws/instance-generation|2| @@ -4529,6 +4702,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|d2| |karpenter.k8s.aws/instance-generation|2| @@ -4552,6 +4726,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|36| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|d2| |karpenter.k8s.aws/instance-generation|2| @@ -4577,6 +4752,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3| |karpenter.k8s.aws/instance-generation|3| @@ -4603,6 +4779,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3| |karpenter.k8s.aws/instance-generation|3| @@ -4629,6 +4806,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3| |karpenter.k8s.aws/instance-generation|3| @@ -4655,6 +4833,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|5000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3| |karpenter.k8s.aws/instance-generation|3| @@ -4682,6 +4861,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4708,6 +4888,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4734,6 +4915,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4760,6 +4942,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4786,6 +4969,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|5000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4812,6 +4996,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4839,6 +5024,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|dl| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|dl1| |karpenter.k8s.aws/instance-generation|1| @@ -4872,13 +5058,13 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|f| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1700| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|f1| |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-local-nvme|470| |karpenter.k8s.aws/instance-memory|124928| - |karpenter.k8s.aws/instance-network-bandwidth|2500| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4897,13 +5083,13 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|f| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|f1| |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-local-nvme|940| |karpenter.k8s.aws/instance-memory|249856| - |karpenter.k8s.aws/instance-network-bandwidth|5000| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4922,6 +5108,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|f| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|f1| |karpenter.k8s.aws/instance-generation|1| @@ -4948,6 +5135,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g3| |karpenter.k8s.aws/instance-generation|3| @@ -4957,7 +5145,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-gpu-name|m60| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|124928| - |karpenter.k8s.aws/instance-network-bandwidth|5000| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4970,6 +5157,7 @@ below are the resources available with some assumptions and after the instance o |memory|112629Mi| |nvidia.com/gpu|1| |pods|234| + |vpc.amazonaws.com/pod-eni|6| ### `g3.8xlarge` #### Labels | Label | Value | @@ -4977,6 +5165,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g3| |karpenter.k8s.aws/instance-generation|3| @@ -4999,6 +5188,7 @@ below are the resources available with some assumptions and after the instance o |memory|228187Mi| |nvidia.com/gpu|2| |pods|234| + |vpc.amazonaws.com/pod-eni|6| ### `g3.16xlarge` #### Labels | Label | Value | @@ -5006,6 +5196,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g3| |karpenter.k8s.aws/instance-generation|3| @@ -5036,6 +5227,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|850| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g3s| |karpenter.k8s.aws/instance-generation|3| @@ -5057,6 +5249,7 @@ below are the resources available with some assumptions and after the instance o |memory|27896Mi| |nvidia.com/gpu|1| |pods|58| + |vpc.amazonaws.com/pod-eni|10| ## g4ad Family ### `g4ad.xlarge` #### Labels @@ -5065,6 +5258,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -5096,6 +5290,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -5127,6 +5322,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -5158,6 +5354,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -5189,6 +5386,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6300| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -5221,6 +5419,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5252,6 +5451,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5283,6 +5483,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5314,6 +5515,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5346,6 +5548,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5378,6 +5581,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5410,6 +5614,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5443,6 +5648,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5474,6 +5680,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5505,6 +5712,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5536,6 +5744,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|16000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5568,6 +5777,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|16000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5600,6 +5810,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|16000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5632,6 +5843,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5664,6 +5876,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5697,6 +5910,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5727,6 +5941,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5757,6 +5972,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5787,6 +6003,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5817,6 +6034,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5847,6 +6065,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5878,6 +6097,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|5000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -5888,6 +6108,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|250| |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|2500| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5900,6 +6121,7 @@ below are the resources available with some assumptions and after the instance o |memory|14162Mi| |nvidia.com/gpu|1| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `g6.2xlarge` #### Labels | Label | Value | @@ -5907,6 +6129,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|5000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -5917,6 +6140,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|450| |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|5000| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5929,6 +6153,7 @@ below are the resources available with some assumptions and after the instance o |memory|29317Mi| |nvidia.com/gpu|1| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `g6.4xlarge` #### Labels | Label | Value | @@ -5936,6 +6161,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|8000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -5946,6 +6172,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|10000| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5958,6 +6185,7 @@ below are the resources available with some assumptions and after the instance o |memory|57691Mi| |nvidia.com/gpu|1| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `g6.8xlarge` #### Labels | Label | Value | @@ -5965,6 +6193,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|16000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -5975,6 +6204,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5988,6 +6218,7 @@ below are the resources available with some assumptions and after the instance o |nvidia.com/gpu|1| |pods|234| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|84| ### `g6.12xlarge` #### Labels | Label | Value | @@ -5995,6 +6226,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -6005,6 +6237,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|15200| |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|40000| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6018,6 +6251,7 @@ below are the resources available with some assumptions and after the instance o |nvidia.com/gpu|4| |pods|234| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|114| ### `g6.16xlarge` #### Labels | Label | Value | @@ -6025,6 +6259,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -6035,6 +6270,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6048,6 +6284,7 @@ below are the resources available with some assumptions and after the instance o |nvidia.com/gpu|1| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `g6.24xlarge` #### Labels | Label | Value | @@ -6055,6 +6292,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -6065,6 +6303,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|15200| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6078,6 +6317,7 @@ below are the resources available with some assumptions and after the instance o |nvidia.com/gpu|4| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `g6.48xlarge` #### Labels | Label | Value | @@ -6085,6 +6325,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -6095,6 +6336,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|60800| |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|100000| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6108,6 +6350,7 @@ below are the resources available with some assumptions and after the instance o |nvidia.com/gpu|8| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## gr6 Family ### `gr6.4xlarge` #### Labels @@ -6116,6 +6359,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|gr| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|8000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|gr6| |karpenter.k8s.aws/instance-generation|6| @@ -6126,6 +6370,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|10000| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6138,6 +6383,7 @@ below are the resources available with some assumptions and after the instance o |memory|118312Mi| |nvidia.com/gpu|1| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `gr6.8xlarge` #### Labels | Label | Value | @@ -6145,6 +6391,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|gr| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|16000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|gr6| |karpenter.k8s.aws/instance-generation|6| @@ -6155,6 +6402,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6168,6 +6416,7 @@ below are the resources available with some assumptions and after the instance o |nvidia.com/gpu|1| |pods|234| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|84| ## h1 Family ### `h1.2xlarge` #### Labels @@ -6176,6 +6425,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|h| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|h1| |karpenter.k8s.aws/instance-generation|1| @@ -6200,6 +6450,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|h| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|h1| |karpenter.k8s.aws/instance-generation|1| @@ -6224,6 +6475,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|h| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|h1| |karpenter.k8s.aws/instance-generation|1| @@ -6248,6 +6500,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|h| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|h1| |karpenter.k8s.aws/instance-generation|1| @@ -6273,6 +6526,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|hpc| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|hpc7g| |karpenter.k8s.aws/instance-generation|7| @@ -6283,6 +6537,7 @@ below are the resources available with some assumptions and after the instance o |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| |node.kubernetes.io/instance-type|hpc7g.4xlarge| + |topology.k8s.aws/zone-id|6419929671613507071| #### Resources | Resource | Quantity | |--|--| @@ -6298,6 +6553,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|hpc| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|hpc7g| |karpenter.k8s.aws/instance-generation|7| @@ -6308,6 +6564,7 @@ below are the resources available with some assumptions and after the instance o |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| |node.kubernetes.io/instance-type|hpc7g.8xlarge| + |topology.k8s.aws/zone-id|3124717047704565898| #### Resources | Resource | Quantity | |--|--| @@ -6323,6 +6580,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|hpc| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|hpc7g| |karpenter.k8s.aws/instance-generation|7| @@ -6333,6 +6591,7 @@ below are the resources available with some assumptions and after the instance o |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| |node.kubernetes.io/instance-type|hpc7g.16xlarge| + |topology.k8s.aws/zone-id|4594531912622968525| #### Resources | Resource | Quantity | |--|--| @@ -6443,6 +6702,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|425| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6468,6 +6728,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|850| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6493,6 +6754,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1700| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6518,6 +6780,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6543,6 +6806,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6568,6 +6832,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6593,6 +6858,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|72| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6620,6 +6886,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6646,6 +6913,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6672,6 +6940,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6698,6 +6967,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|12| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6724,6 +6994,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6750,6 +7021,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6777,6 +7049,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6804,6 +7077,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6832,6 +7106,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6858,6 +7133,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6884,6 +7160,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6910,6 +7187,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6936,6 +7214,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6962,6 +7241,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6990,6 +7270,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7015,6 +7296,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7041,6 +7323,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7067,6 +7350,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7093,6 +7377,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7119,6 +7404,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7145,6 +7431,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7171,6 +7458,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7197,6 +7485,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7224,6 +7513,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7252,6 +7542,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -7278,6 +7569,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -7304,6 +7596,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -7330,6 +7623,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -7356,6 +7650,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -7382,6 +7677,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -7413,6 +7709,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf1| |karpenter.k8s.aws/instance-generation|1| @@ -7442,6 +7739,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf1| |karpenter.k8s.aws/instance-generation|1| @@ -7471,6 +7769,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf1| |karpenter.k8s.aws/instance-generation|1| @@ -7500,6 +7799,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf1| |karpenter.k8s.aws/instance-generation|1| @@ -7531,6 +7831,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf2| |karpenter.k8s.aws/instance-generation|2| @@ -7560,6 +7861,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf2| |karpenter.k8s.aws/instance-generation|2| @@ -7589,6 +7891,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf2| |karpenter.k8s.aws/instance-generation|2| @@ -7618,6 +7921,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf2| |karpenter.k8s.aws/instance-generation|2| @@ -7645,6 +7949,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7671,6 +7976,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7697,6 +8003,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7723,6 +8030,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7749,6 +8057,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7775,6 +8084,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -8058,6 +8368,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|450| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -8081,6 +8392,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -8104,6 +8416,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -8127,6 +8440,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -8150,6 +8464,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|40| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -8174,6 +8489,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -8199,6 +8515,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8224,6 +8541,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8249,6 +8567,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8274,6 +8593,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8299,6 +8619,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8324,6 +8645,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8349,6 +8671,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8374,6 +8697,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8399,6 +8723,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8425,6 +8750,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8450,6 +8776,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8475,6 +8802,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8500,6 +8828,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8525,6 +8854,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8550,6 +8880,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8575,6 +8906,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8600,6 +8932,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|13750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8626,6 +8959,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8652,6 +8986,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8678,6 +9013,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8704,6 +9040,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8730,6 +9067,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8756,6 +9094,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8782,6 +9121,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8808,6 +9148,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|13750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8835,6 +9176,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8861,6 +9203,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8887,6 +9230,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8913,6 +9257,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8939,6 +9284,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8965,6 +9311,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8991,6 +9338,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -9017,6 +9365,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -9043,6 +9392,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -9070,6 +9420,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9096,6 +9447,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9122,6 +9474,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9148,6 +9501,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9174,6 +9528,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9200,6 +9555,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9226,6 +9582,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9252,6 +9609,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9279,6 +9637,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9307,6 +9666,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9332,6 +9692,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9357,6 +9718,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9382,6 +9744,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9407,6 +9770,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9432,6 +9796,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9457,6 +9822,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9482,6 +9848,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9508,6 +9875,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9535,6 +9903,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9560,6 +9929,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9585,6 +9955,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9610,6 +9981,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|12| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9635,6 +10007,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9660,6 +10033,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9686,6 +10060,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9713,6 +10088,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9738,6 +10114,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9763,6 +10140,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9788,6 +10166,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9813,6 +10192,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9838,6 +10218,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9863,6 +10244,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9888,6 +10270,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9913,6 +10296,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9938,6 +10322,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9964,6 +10349,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9991,6 +10377,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10016,6 +10403,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10041,6 +10429,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10066,6 +10455,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10091,6 +10481,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10116,6 +10507,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10141,6 +10533,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10166,6 +10559,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10191,6 +10585,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10217,6 +10612,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10243,6 +10639,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10269,6 +10666,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10295,6 +10693,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10321,6 +10720,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10347,6 +10747,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10373,6 +10774,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10399,6 +10801,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10425,6 +10828,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10452,6 +10856,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10477,6 +10882,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10502,6 +10908,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10527,6 +10934,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10552,6 +10960,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10577,6 +10986,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10602,6 +11012,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10627,6 +11038,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10652,6 +11064,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10678,6 +11091,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10705,6 +11119,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10731,6 +11146,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10757,6 +11173,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10783,6 +11200,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10809,6 +11227,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10835,6 +11254,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10861,6 +11281,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10887,6 +11308,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10913,6 +11335,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10940,6 +11363,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10968,6 +11392,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -10994,6 +11419,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11020,6 +11446,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11046,6 +11473,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11072,6 +11500,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11098,6 +11527,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|37500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11124,6 +11554,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|50000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11150,6 +11581,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|75000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11176,6 +11608,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11192,10 +11625,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|480816Mi| - |pods|345| + |memory|480277Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ### `m6idn.metal` #### Labels | Label | Value | @@ -11203,6 +11636,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11219,10 +11653,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|480816Mi| - |pods|345| + |memory|480277Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ## m6in Family ### `m6in.large` #### Labels @@ -11231,6 +11665,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11256,6 +11691,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11281,6 +11717,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11306,6 +11743,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11331,6 +11769,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11356,6 +11795,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|37500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11381,6 +11821,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|50000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11406,6 +11847,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|75000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11431,6 +11873,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11446,10 +11889,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|480816Mi| - |pods|345| + |memory|480277Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ### `m6in.metal` #### Labels | Label | Value | @@ -11457,6 +11900,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11472,10 +11916,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|480816Mi| - |pods|345| + |memory|480277Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ## m7a Family ### `m7a.medium` #### Labels @@ -11484,6 +11928,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11509,6 +11954,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11534,6 +11980,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11559,6 +12006,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11584,6 +12032,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11609,6 +12058,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11634,6 +12084,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11659,6 +12110,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11684,6 +12136,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11709,6 +12162,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11734,6 +12188,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11760,6 +12215,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11787,6 +12243,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11812,6 +12269,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11837,6 +12295,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11862,6 +12321,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11887,6 +12347,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11912,6 +12373,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11937,6 +12399,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11962,6 +12425,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11988,6 +12452,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -12015,6 +12480,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -12041,6 +12507,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -12067,6 +12534,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -12093,6 +12561,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -12119,6 +12588,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -12145,6 +12615,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -12171,6 +12642,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -12197,6 +12669,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -12224,6 +12697,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -12243,6 +12717,7 @@ below are the resources available with some assumptions and after the instance o |memory|233962Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## m7i Family ### `m7i.large` #### Labels @@ -12251,6 +12726,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12276,6 +12752,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12301,6 +12778,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12326,6 +12804,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12351,6 +12830,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12376,6 +12856,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12401,6 +12882,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12426,6 +12908,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12451,6 +12934,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12476,6 +12960,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12502,6 +12987,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12529,6 +13015,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -12554,6 +13041,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -12579,6 +13067,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -12604,6 +13093,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -12629,6 +13119,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -12655,6 +13146,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p2| |karpenter.k8s.aws/instance-generation|2| @@ -12683,6 +13175,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|5000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p2| |karpenter.k8s.aws/instance-generation|2| @@ -12712,6 +13205,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p2| |karpenter.k8s.aws/instance-generation|2| @@ -12742,6 +13236,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p3| |karpenter.k8s.aws/instance-generation|3| @@ -12770,6 +13265,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p3| |karpenter.k8s.aws/instance-generation|3| @@ -12799,6 +13295,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p3| |karpenter.k8s.aws/instance-generation|3| @@ -12829,6 +13326,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|p3dn| |karpenter.k8s.aws/instance-generation|3| @@ -12862,6 +13360,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|p4d| |karpenter.k8s.aws/instance-generation|4| @@ -12895,6 +13394,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|p5| |karpenter.k8s.aws/instance-generation|5| @@ -13045,6 +13545,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|425| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -13069,6 +13570,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|850| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -13093,6 +13595,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1700| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -13117,6 +13620,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -13141,6 +13645,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -13165,6 +13670,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -13190,6 +13696,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13215,6 +13722,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13240,6 +13748,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13265,6 +13774,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13290,6 +13800,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13315,6 +13826,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13340,6 +13852,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13365,6 +13878,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13390,6 +13904,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13416,6 +13931,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13441,6 +13957,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13466,6 +13983,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13491,6 +14009,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13516,6 +14035,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13541,6 +14061,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13566,6 +14087,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13591,6 +14113,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|13570| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13617,6 +14140,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13643,6 +14167,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13669,6 +14194,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13695,6 +14221,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13721,6 +14248,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13747,6 +14275,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13773,6 +14302,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13799,6 +14329,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|13570| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13826,6 +14357,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13851,6 +14383,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13876,6 +14409,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13901,6 +14435,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13926,6 +14461,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13951,6 +14487,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13976,6 +14513,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -14001,6 +14539,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -14026,6 +14565,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -14052,6 +14592,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14078,6 +14619,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14104,6 +14646,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14130,6 +14673,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14156,6 +14700,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14182,6 +14727,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14208,6 +14754,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14234,6 +14781,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14260,6 +14808,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14287,6 +14836,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14313,6 +14863,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14339,6 +14890,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14365,6 +14917,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14391,6 +14944,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14417,6 +14971,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14443,6 +14998,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14469,6 +15025,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14496,6 +15053,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14524,6 +15082,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14549,6 +15108,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14574,6 +15134,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14599,6 +15160,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14624,6 +15186,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14649,6 +15212,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14674,6 +15238,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14699,6 +15264,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14725,6 +15291,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14752,6 +15319,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14777,6 +15345,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14802,6 +15371,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14827,6 +15397,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14852,6 +15423,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14877,6 +15449,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14902,6 +15475,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14927,6 +15501,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14952,6 +15527,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14977,6 +15553,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -15003,6 +15580,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -15030,6 +15608,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15055,6 +15634,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15080,6 +15660,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15105,6 +15686,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15130,6 +15712,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15155,6 +15738,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15180,6 +15764,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15205,6 +15790,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15230,6 +15816,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15256,6 +15843,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15282,6 +15870,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15308,6 +15897,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15334,6 +15924,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15360,6 +15951,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15386,6 +15978,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15412,6 +16005,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15438,6 +16032,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15464,6 +16059,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15491,6 +16087,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15516,6 +16113,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15541,6 +16139,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15566,6 +16165,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15591,6 +16191,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15616,6 +16217,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15641,6 +16243,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15666,6 +16269,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15691,6 +16295,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15717,6 +16322,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15744,6 +16350,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15770,6 +16377,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15796,6 +16404,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15822,6 +16431,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15848,6 +16458,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15874,6 +16485,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15900,6 +16512,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15926,6 +16539,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15952,6 +16566,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15979,6 +16594,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -16007,6 +16623,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16033,6 +16650,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16059,6 +16677,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16085,6 +16704,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16111,6 +16731,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16137,6 +16758,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|37500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16163,6 +16785,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|50000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16189,6 +16812,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|75000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16215,6 +16839,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16231,10 +16856,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|965782Mi| - |pods|345| + |memory|965243Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ### `r6idn.metal` #### Labels | Label | Value | @@ -16242,6 +16867,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16258,10 +16884,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|965782Mi| - |pods|345| + |memory|965243Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ## r6in Family ### `r6in.large` #### Labels @@ -16270,6 +16896,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16295,6 +16922,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16320,6 +16948,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16345,6 +16974,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16370,6 +17000,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16395,6 +17026,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|37500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16420,6 +17052,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|50000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16445,6 +17078,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|75000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16470,6 +17104,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16485,10 +17120,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|965782Mi| - |pods|345| + |memory|965243Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ### `r6in.metal` #### Labels | Label | Value | @@ -16496,6 +17131,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16511,10 +17147,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|965782Mi| - |pods|345| + |memory|965243Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ## r7a Family ### `r7a.medium` #### Labels @@ -16523,6 +17159,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16548,6 +17185,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16573,6 +17211,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16598,6 +17237,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16623,6 +17263,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16648,6 +17289,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16673,6 +17315,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16698,6 +17341,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16723,6 +17367,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16748,6 +17393,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16773,6 +17419,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16799,6 +17446,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16826,6 +17474,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -16851,6 +17500,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -16876,6 +17526,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -16901,6 +17552,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -16926,6 +17578,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -16951,6 +17604,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -16976,6 +17630,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -17001,6 +17656,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -17027,6 +17683,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -17054,6 +17711,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17080,6 +17738,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17106,6 +17765,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17132,6 +17792,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17158,6 +17819,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17184,6 +17846,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17210,6 +17873,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17236,6 +17900,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17263,6 +17928,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17282,6 +17948,7 @@ below are the resources available with some assumptions and after the instance o |memory|476445Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## r7i Family ### `r7i.large` #### Labels @@ -17290,6 +17957,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17315,6 +17983,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17340,6 +18009,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17365,6 +18035,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17390,6 +18061,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17415,6 +18087,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17440,6 +18113,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17465,6 +18139,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17490,6 +18165,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17515,6 +18191,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17541,6 +18218,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17568,6 +18246,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17593,6 +18272,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17618,6 +18298,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17643,6 +18324,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17668,6 +18350,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17693,6 +18376,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17718,6 +18402,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17743,6 +18428,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17768,6 +18454,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17794,6 +18481,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -18007,6 +18695,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18031,6 +18720,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18055,6 +18745,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18079,6 +18770,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18103,6 +18795,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18127,6 +18820,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18151,6 +18845,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18176,6 +18871,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18200,6 +18896,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18224,6 +18921,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18248,6 +18946,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18272,6 +18971,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18296,6 +18996,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18320,6 +19021,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18345,6 +19047,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18369,6 +19072,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18393,6 +19097,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18417,6 +19122,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18441,6 +19147,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18465,6 +19172,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18489,6 +19197,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18517,6 +19226,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|trn| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|trn1| |karpenter.k8s.aws/instance-generation|1| @@ -18547,6 +19257,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|trn| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|trn1| |karpenter.k8s.aws/instance-generation|1| @@ -18579,6 +19290,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|trn| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|trn1n| |karpenter.k8s.aws/instance-generation|1| @@ -18608,6 +19320,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-12tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18633,6 +19346,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-18tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18658,6 +19372,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-24tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18683,6 +19398,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|224| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-3tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18709,6 +19425,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|224| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-6tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18733,6 +19450,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-6tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18758,6 +19476,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-9tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18775,6 +19494,110 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|8720933Mi| |pods|737| +## u7i-12tb Family +### `u7i-12tb.224xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|u| + |karpenter.k8s.aws/instance-cpu|896| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|u7i-12tb| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|12582912| + |karpenter.k8s.aws/instance-size|224xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|u7i-12tb.224xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|893690m| + |ephemeral-storage|17Gi| + |memory|11630731Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| +## u7in-16tb Family +### `u7in-16tb.224xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|u| + |karpenter.k8s.aws/instance-cpu|896| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|u7in-16tb| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|16777216| + |karpenter.k8s.aws/instance-size|224xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|u7in-16tb.224xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|893690m| + |ephemeral-storage|17Gi| + |memory|15514235Mi| + |pods|394| + |vpc.amazonaws.com/efa|2| +## u7in-24tb Family +### `u7in-24tb.224xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|u| + |karpenter.k8s.aws/instance-cpu|896| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|u7in-24tb| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|25165824| + |karpenter.k8s.aws/instance-size|224xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|u7in-24tb.224xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|893690m| + |ephemeral-storage|17Gi| + |memory|23273698Mi| + |pods|394| + |vpc.amazonaws.com/efa|2| +## u7in-32tb Family +### `u7in-32tb.224xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|u| + |karpenter.k8s.aws/instance-cpu|896| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|u7in-32tb| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|33554432| + |karpenter.k8s.aws/instance-size|224xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|u7in-32tb.224xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|893690m| + |ephemeral-storage|17Gi| + |memory|31033160Mi| + |pods|394| + |vpc.amazonaws.com/efa|2| ## vt1 Family ### `vt1.3xlarge` #### Labels @@ -18783,6 +19606,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|vt| |karpenter.k8s.aws/instance-cpu|12| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|vt1| |karpenter.k8s.aws/instance-generation|1| @@ -18808,6 +19632,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|vt| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|vt1| |karpenter.k8s.aws/instance-generation|1| @@ -18833,6 +19658,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|vt| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|vt1| |karpenter.k8s.aws/instance-generation|1| @@ -18860,6 +19686,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1| |karpenter.k8s.aws/instance-generation|1| @@ -18884,6 +19711,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1| |karpenter.k8s.aws/instance-generation|1| @@ -18909,6 +19737,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -18933,6 +19762,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -18957,6 +19787,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -18981,6 +19812,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -19005,6 +19837,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -19029,6 +19862,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -19054,6 +19888,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19080,6 +19915,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19106,6 +19942,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19132,6 +19969,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19158,6 +19996,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19184,6 +20023,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19210,6 +20050,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19236,6 +20077,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19262,6 +20104,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19289,6 +20132,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2idn| |karpenter.k8s.aws/instance-generation|2| @@ -19315,6 +20159,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2idn| |karpenter.k8s.aws/instance-generation|2| @@ -19341,6 +20186,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2idn| |karpenter.k8s.aws/instance-generation|2| @@ -19368,6 +20214,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2idn| |karpenter.k8s.aws/instance-generation|2| @@ -19396,6 +20243,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19422,6 +20270,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19448,6 +20297,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19474,6 +20324,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19500,6 +20351,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19526,6 +20378,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19552,6 +20405,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19579,6 +20433,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19607,6 +20462,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -19632,6 +20488,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -19657,6 +20514,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -19682,6 +20540,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|12000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -19707,6 +20566,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -19733,6 +20593,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -19760,6 +20621,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -19786,6 +20648,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -19812,6 +20675,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -19838,6 +20702,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|12| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -19864,6 +20729,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -19890,6 +20756,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -19916,6 +20783,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| diff --git a/website/content/en/docs/reference/metrics.md b/website/content/en/docs/reference/metrics.md index c717023a2cad..30e52c681726 100644 --- a/website/content/en/docs/reference/metrics.md +++ b/website/content/en/docs/reference/metrics.md @@ -7,7 +7,7 @@ description: > Inspect Karpenter Metrics --- -Karpenter makes several metrics available in Prometheus format to allow monitoring cluster provisioning status. These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8000/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings) +Karpenter makes several metrics available in Prometheus format to allow monitoring cluster provisioning status. These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8080/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings) ### `karpenter_build_info` A metric with a constant '1' value labeled by version from which karpenter was built. diff --git a/website/content/en/docs/reference/settings.md b/website/content/en/docs/reference/settings.md index 753e9ef7b707..2ce600fb8e08 100644 --- a/website/content/en/docs/reference/settings.md +++ b/website/content/en/docs/reference/settings.md @@ -23,7 +23,7 @@ Karpenter surfaces environment variables and CLI parameters to allow you to conf | ENABLE_PROFILING | \-\-enable-profiling | Enable the profiling on the metric endpoint| | FEATURE_GATES | \-\-feature-gates | Optional features can be enabled / disabled using feature gates. Current options are: Drift,SpotToSpotConsolidation (default = Drift=true,SpotToSpotConsolidation=false)| | HEALTH_PROBE_PORT | \-\-health-probe-port | The port the health probe endpoint binds to for reporting controller health (default = 8081)| -| INTERRUPTION_QUEUE | \-\-interruption-queue | Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.| +| INTERRUPTION_QUEUE | \-\-interruption-queue | Interruption queue is the name of the SQS queue used for processing interruption events from EC2. Interruption handling is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.| | ISOLATED_VPC | \-\-isolated-vpc | If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS on-demand pricing endpoint.| | KARPENTER_SERVICE | \-\-karpenter-service | The Karpenter Service name for the dynamic webhook certificate| | KUBE_CLIENT_BURST | \-\-kube-client-burst | The maximum allowed burst of queries to the kube-apiserver (default = 300)| @@ -31,7 +31,7 @@ Karpenter surfaces environment variables and CLI parameters to allow you to conf | LEADER_ELECT | \-\-leader-elect | Start leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.| | LOG_LEVEL | \-\-log-level | Log verbosity level. Can be one of 'debug', 'info', or 'error' (default = info)| | MEMORY_LIMIT | \-\-memory-limit | Memory limit on the container running the controller. The GC soft memory limit is set to 90% of this value. (default = -1)| -| METRICS_PORT | \-\-metrics-port | The port the metric endpoint binds to for operating metrics about the controller itself (default = 8000)| +| METRICS_PORT | \-\-metrics-port | The port the metric endpoint binds to for operating metrics about the controller itself (default = 8080)| | RESERVED_ENIS | \-\-reserved-enis | Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html. (default = 0)| | VM_MEMORY_OVERHEAD_PERCENT | \-\-vm-memory-overhead-percent | The VM memory overhead as a percent that will be subtracted from the total memory for all instance types. (default = 0.075)| | WEBHOOK_METRICS_PORT | \-\-webhook-metrics-port | The port the webhook metric endpoing binds to for operating metrics about the webhook (default = 8001)| diff --git a/website/content/en/docs/reference/threat-model.md b/website/content/en/docs/reference/threat-model.md index 71f8beaf3532..63df939b79a6 100644 --- a/website/content/en/docs/reference/threat-model.md +++ b/website/content/en/docs/reference/threat-model.md @@ -31,11 +31,11 @@ A Cluster Developer has the ability to create pods via `Deployments`, `ReplicaSe Karpenter has permissions to create and manage cloud instances. Karpenter has Kubernetes API permissions to create, update, and remove nodes, as well as evict pods. For a full list of the permissions, see the RBAC rules in the helm chart template. Karpenter also has AWS IAM permissions to create instances with IAM roles. -* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.36.0/charts/karpenter/templates/aggregate-clusterrole.yaml) -* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.36.0/charts/karpenter/templates/clusterrole-core.yaml) -* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.36.0/charts/karpenter/templates/clusterrole.yaml) -* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.36.0/charts/karpenter/templates/rolebinding.yaml) -* [role.yaml](https://github.com/aws/karpenter/blob/v0.36.0/charts/karpenter/templates/role.yaml) +* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.37.0/charts/karpenter/templates/aggregate-clusterrole.yaml) +* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.37.0/charts/karpenter/templates/clusterrole-core.yaml) +* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.37.0/charts/karpenter/templates/clusterrole.yaml) +* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.37.0/charts/karpenter/templates/rolebinding.yaml) +* [role.yaml](https://github.com/aws/karpenter/blob/v0.37.0/charts/karpenter/templates/role.yaml) ## Assumptions diff --git a/website/content/en/docs/troubleshooting.md b/website/content/en/docs/troubleshooting.md index 6dc784007b75..6a5ec49e6a5a 100644 --- a/website/content/en/docs/troubleshooting.md +++ b/website/content/en/docs/troubleshooting.md @@ -73,7 +73,7 @@ Node role names for Karpenter are created in the form `KarpenterNodeRole-${Clust If a long cluster name causes the Karpenter node role name to exceed 64 characters, creating that object will fail. Keep in mind that `KarpenterNodeRole-` is just a recommendation from the getting started guide. -Instead using of the eksctl role, you can shorten the name to anything you like, as long as it has the right permissions. +Instead of using the eksctl role, you can shorten the name to anything you like, as long as it has the right permissions. ### Unknown field in Provisioner spec @@ -117,14 +117,16 @@ Karpenter `0.26.1` introduced the `karpenter-crd` Helm chart. When installing th - In the case of `invalid ownership metadata; label validation error: missing key "app.kubernetes.io/managed-by": must be set to "Helm"` run: ```shell +KARPENTER_NAMESPACE=kube-system kubectl label crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh app.kubernetes.io/managed-by=Helm --overwrite ``` - In the case of `annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "karpenter"` run: ```shell +KARPENTER_NAMESPACE=kube-system kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite -kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace="${KARPENTER_NAMESPACE}" --overwrite ``` ## Uninstallation @@ -328,7 +330,12 @@ time=2023-06-12T19:18:15Z type=Warning reason=FailedCreatePodSandBox from=kubele By default, the number of pods on a node is limited by both the number of networking interfaces (ENIs) that may be attached to an instance type and the number of IP addresses that can be assigned to each ENI. See [IP addresses per network interface per instance type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI) for a more detailed information on these instance types' limits. -If the max-pods (configured through your Provisioner [`kubeletConfiguration`]({{}})) is greater than the number of supported IPs for a given instance type, the CNI will fail to assign an IP to the pod and your pod will be left in a `ContainerCreating` state. +If the max-pods (configured through your NodePool [`kubeletConfiguration`]({{}})) is greater than the number of supported IPs for a given instance type, the CNI will fail to assign an IP to the pod and your pod will be left in a `ContainerCreating` state. + +If you've enabled [Security Groups per Pod](https://aws.github.io/aws-eks-best-practices/networking/sgpp/), one of the instance's ENIs is reserved as the trunk interface and uses branch interfaces off of that trunk interface to assign different security groups. +If you do not have any `SecurityGroupPolicies` configured for your pods, they will be unable to utilize branch interfaces attached to the trunk interface, and IPs will only be available from the non-trunk ENIs. +This effectively reduces the max-pods value by the number of IPs that would have been available from the trunk ENI. +Note that Karpenter is not aware if [Security Groups per Pod](https://aws.github.io/aws-eks-best-practices/networking/sgpp/) is enabled, and will continue to compute max-pods assuming all ENIs on the instance can be utilized. ##### Solutions @@ -647,7 +654,7 @@ To correct the problem if it occurs, you can use the approach that AWS EBS uses, "AWS": "arn:aws:iam::${AWS_ACCOUNT_ID}:root" }, "Action": [ - "kms:Describe", + "kms:Describe*", "kms:Get*", "kms:List*", "kms:RevokeGrant" @@ -663,7 +670,7 @@ This typically occurs when the node has not been considered fully initialized fo ### Log message of `inflight check failed for node, Expected resource "vpc.amazonaws.com/pod-eni" didn't register on the node` is reported -This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. If you've enabled Pod ENI for Karpenter nodes via the `aws.enablePodENI` setting, you will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. +This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. You will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. ### AWS Node Termination Handler (NTH) interactions Karpenter [doesn't currently support draining and terminating on spot rebalance recommendations]({{< ref "concepts/disruption#interruption" >}}). Users who want support for both drain and terminate on spot interruption as well as drain and termination on spot rebalance recommendations may install Node Termination Handler (NTH) on their clusters to support this behavior. diff --git a/website/content/en/docs/upgrading/compatibility.md b/website/content/en/docs/upgrading/compatibility.md index 0f2c8d6cab94..2ccb7d2c9d1e 100644 --- a/website/content/en/docs/upgrading/compatibility.md +++ b/website/content/en/docs/upgrading/compatibility.md @@ -15,9 +15,9 @@ Before you begin upgrading Karpenter, consider Karpenter compatibility issues re [comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) -| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | 1.29 | -|------------|----------|----------|----------|----------|----------|----------|------------| -| karpenter | \>= 0.21 | \>= 0.21 | \>= 0.25 | \>= 0.28 | \>= 0.28 | \>= 0.31 | \>= 0.34.0 | +| KUBERNETES | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | 1.29 | 1.30 | +|------------|----------|----------|----------|----------|----------|----------|--------| +| karpenter | \>= 0.21 | \>= 0.25 | \>= 0.28 | \>= 0.28 | \>= 0.31 | \>= 0.34 | 0.37.0 | [comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) @@ -75,14 +75,13 @@ Karpenter offers three types of releases. This section explains the purpose of e ### Stable Releases -Stable releases are the most reliable releases that are released with weekly cadence. Stable releases are our only recommended versions for production environments. -Sometimes we skip a stable release because we find instability or problems that need to be fixed before having a stable release. -Stable releases are tagged with a semantic version prefixed by a `v`. For example `v0.13.0`. +Stable releases are the only recommended versions for production environments. Stable releases are tagged with a semantic version (e.g. `0.35.0`). Note that stable releases prior to `0.35.0` are prefixed with a `v` (e.g. `v0.34.0`). ### Release Candidates -We consider having release candidates for major and important minor versions. Our release candidates are tagged like `vx.y.z-rc.0`, `vx.y.z-rc.1`. The release candidate will then graduate to `vx.y.z` as a normal stable release. +We consider having release candidates for major and important minor versions. Our release candidates are tagged like `x.y.z-rc.0`, `x.y.z-rc.1`. The release candidate will then graduate to `x.y.z` as a stable release. By adopting this practice we allow our users who are early adopters to test out new releases before they are available to the wider community, thereby providing us with early feedback resulting in more stable releases. +Note that, like the stable releases, release candidates prior to `0.35.0` are prefixed with a `v`. ### Snapshot Releases diff --git a/website/content/en/docs/upgrading/upgrade-guide.md b/website/content/en/docs/upgrading/upgrade-guide.md index c7e84b894521..0c4624b652d4 100644 --- a/website/content/en/docs/upgrading/upgrade-guide.md +++ b/website/content/en/docs/upgrading/upgrade-guide.md @@ -28,15 +28,26 @@ If you get the error `invalid ownership metadata; label validation error:` while In general, you can reapply the CRDs in the `crds` directory of the Karpenter Helm chart: ```shell -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.36.0/pkg/apis/crds/karpenter.sh_nodepools.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.36.0/pkg/apis/crds/karpenter.sh_nodeclaims.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.36.0/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.37.0/pkg/apis/crds/karpenter.sh_nodepools.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.37.0/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.37.0/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml ``` +### Upgrading to `0.37.0`+ + +{{% alert title="Warning" color="warning" %}} +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to `0.37.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} + +* Karpenter now adds a readiness status condition to the EC2NodeClass. Make sure to upgrade your Custom Resource Definitions before proceeding with the upgrade. Failure to do so will result in Karpenter being unable to provision new nodes. +* Karpenter no longer updates the logger name when creating controller loggers. We now adhere to the controller-runtime standard, where the logger name will be set as `"logger": "controller"` always and the controller name will be stored in the structured value `"controller"` +* Karpenter updated the NodeClass controller naming in the following way: `nodeclass` -> `nodeclass.status`, `nodeclass.hash`, `nodeclass.termination` +* Karpenter's NodeClaim status conditions no longer include the `severity` field + ### Upgrading to `0.36.0`+ {{% alert title="Warning" color="warning" %}} @@ -44,7 +55,7 @@ WHEN CREATING A NEW SECTION OF THE UPGRADE GUIDANCE FOR NEWER VERSIONS, ENSURE T {{% /alert %}} {{% alert title="Warning" color="warning" %}} - v0.36.x introduces update to drift that restricts rollback. When rolling back from >=v0.36.0, note that v0.32.9+, v0.33.4+, v0.34.5+, v0.35.4+ are the patch versions that support rollback. If Karpenter is rolled back to an older patch version, Karpenter can potentially drift all the nodes in the cluster. + v0.36.x introduces update to drift that restricts rollback. When rolling back from >=v0.36.0, note that v0.32.9+, v0.33.4+, v0.34.5+, v0.35.4+ are the patch versions that support rollback. If Karpenter is rolled back to an older patch version, Karpenter can potentially drift all the nodes in the cluster. {{% /alert %}} * Karpenter changed the name of the `karpenter_cloudprovider_instance_type_price_estimate` metric to `karpenter_cloudprovider_instance_type_offering_price_estimate` to align with the new `karpenter_cloudprovider_instance_type_offering_available` metric. The `region` label was also dropped from the metric, since this can be inferred from the environment that Karpenter is running in. @@ -72,7 +83,7 @@ The Ubuntu EKS optimized AMI has moved from 20.04 to 22.04 for Kubernetes 1.29+. * `Empty Expiration / Empty Drift / Empty Consolidation`: infinite parallelism * `Non-Empty Expiration / Non-Empty Drift / Single-Node Consolidation`: one node at a time * `Multi-Node Consolidation`: max 100 nodes -* To support Disruption Budgets, `0.34.0`+ includes critical changes to Karpenter's core controllers, which allows Karpenter to consider multiple batches of disrupting nodes simultaneously. This increases Karpenter's performance with the potential downside of higher CPU and memory utilization from the Karpenter pod. While the magnitude of this difference varies on a case-by-case basis, when upgrading to Karpenter `0.34.0`+, please note that you may need to increase the resources allocated to the Karpenter controller pods. +* To support Disruption Budgets, `0.34.0`+ includes critical changes to Karpenter's core controllers, which allows Karpenter to consider multiple batches of disrupting nodes simultaneously. This increases Karpenter's performance with the potential downside of higher CPU and memory utilization from the Karpenter pod. While the magnitude of this difference varies on a case-by-case basis, when upgrading to Karpenter `0.34.0`+, please note that you may need to increase the resources allocated to the Karpenter controller pods. * Karpenter now adds a default `podSecurityContext` that configures the `fsgroup: 65536` of volumes in the pod. If you are using sidecar containers, you should review if this configuration is compatible for them. You can disable this default `podSecurityContext` through helm by performing `--set podSecurityContext=null` when installing/upgrading the chart. * The `dnsPolicy` for the Karpenter controller pod has been changed back to the Kubernetes cluster default of `ClusterFirst`. Setting our `dnsPolicy` to `Default` (confusingly, this is not the Kubernetes cluster default) caused more confusion for any users running IPv6 clusters with dual-stack nodes or anyone running Karpenter with dependencies on cluster services (like clusters running service meshes). This change may be breaking for any users on Fargate or MNG who were allowing Karpenter to manage their in-cluster DNS service (`core-dns` on most clusters). If you still want the old behavior here, you can change the `dnsPolicy` to point to use `Default` by setting the helm value on install/upgrade with `--set dnsPolicy=Default`. More details on this issue can be found in the following Github issues: [#2186](https://github.com/aws/karpenter-provider-aws/issues/2186) and [#4947](https://github.com/aws/karpenter-provider-aws/issues/4947). * Karpenter now disallows `nodepool.spec.template.spec.resources` to be set. The webhook validation never allowed `nodepool.spec.template.spec.resources`. We are now ensuring that CEL validation also disallows `nodepool.spec.template.spec.resources` to be set. If you were previously setting the resources field on your NodePool, ensure that you remove this field before upgrading to the newest version of Karpenter or else updates to the resource may fail on the new version. @@ -100,9 +111,10 @@ Karpenter `0.32.0` introduces v1beta1 APIs, including _significant_ changes to t This version includes **dual support** for both alpha and beta APIs to ensure that you can slowly migrate your existing Provisioner, AWSNodeTemplate, and Machine alpha APIs to the newer NodePool, EC2NodeClass, and NodeClaim beta APIs. -Note that if you are rolling back after upgrading to `0.32.0`, note that `0.31.4` is the only version that supports handling rollback after you have deployed the v1beta1 APIs to your cluster. +Note that if you are rolling back after upgrading to `0.32.0`, note that __only__ versions `0.31.4` support handling rollback after you have deployed the v1beta1 APIs to your cluster. {{% /alert %}} +* Karpenter now uses `settings.InterruptionQueue` instead of `settings.aws.InterruptionQueueName` in its helm chart. The CLI argument also changed to `--interruption-queue`. * Karpenter now serves the webhook prometheus metrics server on port `8001`. If this port is already in-use on the pod or you are running in `hostNetworking` mode, you may need to change this port value. You can configure this port value through the `WEBHOOK_METRICS_PORT` environment variable or the `webhook.metrics.port` value if installing via Helm. * Karpenter now exposes the ability to disable webhooks through the `webhook.enabled=false` value. This value will disable the webhook server and will prevent any permissions, mutating or validating webhook configurations from being deployed to the cluster. * Karpenter now moves all logging configuration for the Zap logger into the `logConfig` values block. Configuring Karpenter logging with this mechanism _is_ deprecated and will be dropped at v1. Karpenter now only surfaces logLevel through the `logLevel` helm value. If you need more advanced configuration due to log parsing constraints, we recommend configuring your log parser to handle Karpenter's Zap JSON logging. diff --git a/website/content/en/preview/concepts/_index.md b/website/content/en/preview/concepts/_index.md index 67d573135369..e7f8ce4cb500 100755 --- a/website/content/en/preview/concepts/_index.md +++ b/website/content/en/preview/concepts/_index.md @@ -29,7 +29,7 @@ Once privileges are in place, Karpenter is deployed with a Helm chart. ### Configuring NodePools -Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. To configure Karpenter, you create [NodePools]({{}}) that define how Karpenter manages unschedulable pods and configures nodes. You will also define behaviors for your NodePools, capturing details like how Karpenter handles disruption of nodes and setting limits and weights for each NodePool +Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. To configure Karpenter, you create [NodePools]({{}}) that define how Karpenter manages unschedulable pods and configures nodes. You will also define behaviors for your NodePools, capturing details like how Karpenter handles disruption of nodes and setting limits and weights for each NodePool. Here are some things to know about Karpenter's NodePools: diff --git a/website/content/en/preview/concepts/disruption.md b/website/content/en/preview/concepts/disruption.md index 3dcbf0942779..bb6744c46d46 100644 --- a/website/content/en/preview/concepts/disruption.md +++ b/website/content/en/preview/concepts/disruption.md @@ -1,7 +1,7 @@ --- title: "Disruption" linkTitle: "Disruption" -weight: 4 +weight: 50 description: > Understand different ways Karpenter disrupts nodes --- @@ -13,7 +13,7 @@ The finalizer blocks deletion of the node object while the Termination Controlle ### Disruption Controller -Karpenter automatically discovers disruptable nodes and spins up replacements when needed. Karpenter disrupts nodes by executing one [automated method](#automated-methods) at a time, in order of Expiration, Drift, and then Consolidation. Each method varies slightly, but they all follow the standard disruption process. Karpenter uses [disruption budgets]({{}}) to control the speed of disruption. +Karpenter automatically discovers disruptable nodes and spins up replacements when needed. Karpenter disrupts nodes by executing one [automated method](#automated-methods) at a time, first doing Drift then Consolidation. Each method varies slightly, but they all follow the standard disruption process. Karpenter uses [disruption budgets]({{}}) to control the speed at which these disruptions begin. 1. Identify a list of prioritized candidates for the disruption method. * If there are [pods that cannot be evicted](#pod-eviction) on the node, Karpenter will ignore the node and try disrupting it later. * If there are no disruptable nodes, continue to the next disruption method. @@ -61,11 +61,10 @@ By adding the finalizer, Karpenter improves the default Kubernetes process of no When you run `kubectl delete node` on a node without a finalizer, the node is deleted without triggering the finalization logic. The instance will continue running in EC2, even though there is no longer a node object for it. The kubelet isn’t watching for its own existence, so if a node is deleted, the kubelet doesn’t terminate itself. All the pod objects get deleted by a garbage collection process later, because the pods’ node is gone. {{% /alert %}} -## Automated Methods +## Automated Graceful Methods -Automated methods can be rate limited through [NodePool Disruption Budgets]({{}}) +Automated graceful methods, can be rate limited through [NodePool Disruption Budgets]({{}}) -* **Expiration**: Karpenter will mark nodes as expired and disrupt them after they have lived a set number of seconds, based on the NodePool's `spec.disruption.expireAfter` value. You can use node expiry to periodically recycle nodes due to security concerns. * [**Consolidation**]({{}}): Karpenter works to actively reduce cluster cost by identifying when: * Nodes can be removed because the node is empty * Nodes can be removed as their workloads will run on other nodes in the cluster. @@ -74,22 +73,22 @@ Automated methods can be rate limited through [NodePool Disruption Budgets]({{}}): Karpenter will watch for upcoming interruption events that could affect your nodes (health events, spot interruption, etc.) and will taint, drain, and terminate the node(s) ahead of the event to reduce workload disruption. {{% alert title="Defaults" color="secondary" %}} -Disruption is configured through the NodePool's disruption block by the `consolidationPolicy`, `expireAfter` and `consolidateAfter` fields. Karpenter will configure these fields with the following values by default if they are not set: +Disruption is configured through the NodePool's disruption block by the `consolidationPolicy`, and `consolidateAfter` fields. `expireAfter` can also be used to control disruption. Karpenter will configure these fields with the following values by default if they are not set: ```yaml spec: disruption: - consolidationPolicy: WhenUnderutilized - expireAfter: 720h + consolidationPolicy: WhenEmptyOrUnderutilized + template: + spec: + expireAfter: 720h ``` {{% /alert %}} -{{% alert title="Warning" color="warning" %}} -`consolidateAfter` **cannot** be set if `consolidationPolicy` is set to `WhenUnderutilized`. See [kubernetes-sigs/karpenter#735](https://github.com/kubernetes-sigs/karpenter/issues/735) for more information. -{{% /alert %}} - ### Consolidation +Consolidation is configured by `consolidationPolicy` and `consolidateAfter`. `consolidationPolicy` determines the pre-conditions for nodes to be considered consolidatable, and are `whenEmpty` or `whenEmptyOrUnderutilized`. If a node has no running non-daemon pods, it is considered empty. `consolidateAfter` can be set to indicate how long Karpenter should wait after a pod schedules or is removed from the node before considering the node consolidatable. With `whenEmptyOrUnderutilized`, Karpenter will consider a node consolidatable when its `consolidateAfter` has been reached, empty or not. + Karpenter has two mechanisms for cluster consolidation: 1. **Deletion** - A node is eligible for deletion if all of its pods can run on free capacity of other nodes in the cluster. 2. **Replace** - A node can be replaced if all of its pods can run on a combination of free capacity of other nodes in the cluster and a single lower price replacement node. @@ -169,6 +168,13 @@ Karpenter will add the `Drifted` status condition on NodeClaims if the NodeClaim 1. The `Drift` feature gate is not enabled but the NodeClaim is drifted, Karpenter will remove the status condition. 2. The NodeClaim isn't drifted, but has the status condition, Karpenter will remove it. +## Automated Forceful Methods + +Automated forceful methods will begin draining nodes as soon as the condition is met. Note that these methods blow past NodePool Disruption Budgets, and do not wait for a pre-spin replacement node to be healthy for the pods to reschedule, unlike the graceful methods mentioned above. Use Pod Disruption Budgets and `do-not-disrupt` on your nodes to rate-limit the speed at which your applications are disrupted. + +### Expiration +Karpenter will disrupt nodes as soon as they're expired after they've lived for the duration of the NodePool's `spec.template.spec.expireAfter`. You can use expiration to periodically recycle nodes due to security concern. + ### Interruption If interruption-handling is enabled, Karpenter will watch for upcoming involuntary interruption events that would cause disruption to your workloads. These interruption events include: @@ -190,13 +196,24 @@ If you require handling for Spot Rebalance Recommendations, you can use the [AWS Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). -To enable interruption handling, configure the `--interruption-queue-name` CLI argument with the name of the interruption queue provisioned to handle interruption events. +To enable interruption handling, configure the `--interruption-queue` CLI argument with the name of the interruption queue provisioned to handle interruption events. ## Controls -### Disruption Budgets +### TerminationGracePeriod + +This is the duration of time that a node can be draining before it's forcibly deleted. A node begins draining when it's deleted. Pods will be deleted preemptively based on its TerminationGracePeriodSeconds before this terminationGracePeriod ends to give as much time to cleanup as possible. Note that if your pod's terminationGracePeriodSeconds is larger than this terminationGracePeriod, Karpenter may forcibly delete the pod before it has its full terminationGracePeriod to cleanup. + +This is especially useful in combination with `nodepool.spec.template.spec.expireAfter` to define an absolute maximum on the lifetime of a node, where a node is deleted at `expireAfter` and finishes draining within the `terminationGracePeriod` thereafter. Pods blocking eviction like PDBs and do-not-disrupt will block full draining until the `terminationGracePeriod` is reached. + +For instance, a NodeClaim with `terminationGracePeriod` set to `1h` and an `expireAfter` set to `23h` will begin draining after it's lived for `23h`. Let's say a `do-not-disrupt` pod has `TerminationGracePeriodSeconds` set to `300` seconds. If the node hasn't been fully drained after `55m`, Karpenter will delete the pod to allow it's full `terminationGracePeriodSeconds` to cleanup. If no pods are blocking draining, Karpenter will cleanup the node as soon as the node is fully drained, rather than waiting for the NodeClaim's `terminationGracePeriod` to finish. -You can rate limit Karpenter's disruption through the NodePool's `spec.disruption.budgets`. If undefined, Karpenter will default to one budget with `nodes: 10%`. Budgets will consider nodes that are actively being deleted for any reason, and will only block Karpenter from disrupting nodes voluntarily through expiration, drift, emptiness, and consolidation. +### NodePool Disruption Budgets + +You can rate limit Karpenter's disruption through the NodePool's `spec.disruption.budgets`. If undefined, Karpenter will default to one budget with `nodes: 10%`. Budgets will consider nodes that are actively being deleted for any reason, and will only block Karpenter from disrupting nodes voluntarily through drift, emptiness, and consolidation. Note that NodePool Disruption Budgets do not prevent Karpenter from cleaning up expired or drifted nodes. + +#### Reasons +Karpenter allows specifying if a budget applies to any of `Drifted`, `Underutilized`, or `Empty`. When a budget has no reasons, it's assumed that it applies to all reasons. When calculating allowed disruptions for a given reason, Karpenter will take the minimum of the budgets that have listed the reason or have left reasons undefined. #### Nodes When calculating if a budget will block nodes from disruption, Karpenter lists the total number of nodes owned by a NodePool, subtracting out the nodes owned by that NodePool that are currently being deleted and nodes that are NotReady. If the number of nodes being deleted by Karpenter or any other processes is greater than the number of allowed disruptions, disruption for this node will not proceed. @@ -204,30 +221,37 @@ When calculating if a budget will block nodes from disruption, Karpenter lists t If the budget is configured with a percentage value, such as `20%`, Karpenter will calculate the number of allowed disruptions as `allowed_disruptions = roundup(total * percentage) - total_deleting - total_notready`. If otherwise defined as a non-percentage value, Karpenter will simply subtract the number of nodes from the total `(total - non_percentage_value) - total_deleting - total_notready`. For multiple budgets in a NodePool, Karpenter will take the minimum value (most restrictive) of each of the budgets. For example, the following NodePool with three budgets defines the following requirements: -- The first budget will only allow 20% of nodes owned by that NodePool to be disrupted. For instance, if there were 19 nodes owned by the NodePool, 4 disruptions would be allowed, rounding up from `19 * .2 = 3.8`. +- The first budget will only allow 20% of nodes owned by that NodePool to be disrupted if it's empty or drifted. For instance, if there were 19 nodes owned by the NodePool, 4 empty or drifted nodes could be disrupted, rounding up from `19 * .2 = 3.8`. - The second budget acts as a ceiling to the previous budget, only allowing 5 disruptions when there are more than 25 nodes. -- The last budget only blocks disruptions during the first 10 minutes of the day, where 0 disruptions are allowed. +- The last budget only blocks disruptions during the first 10 minutes of the day, where 0 disruptions are allowed, only applying to underutilized nodes. ```yaml -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default spec: + template: + spec: + expireAfter: 720h # 30 * 24h = 720h disruption: - consolidationPolicy: WhenUnderutilized - expireAfter: 720h # 30 * 24h = 720h + consolidationPolicy: WhenEmptyOrUnderutilized budgets: - nodes: "20%" + reasons: + - "empty" + - "drifted" - nodes: "5" - nodes: "0" schedule: "@daily" duration: 10m + reasons: + - "underutilized" ``` #### Schedule Schedule is a cronjob schedule. Generally, the cron syntax is five space-delimited values with options below, with additional special macros like `@yearly`, `@monthly`, `@weekly`, `@daily`, `@hourly`. -Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#writing-a-cronjob-spec) for more information on how to follow the cron syntax. +Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#writing-a-cronjob-spec) for more information on how to follow the cron syntax. Timezones are not currently supported. Schedules are always in UTC. ```bash # ┌───────────── minute (0 - 59) @@ -241,10 +265,6 @@ Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/worklo # * * * * * ``` -{{% alert title="Note" color="primary" %}} -Timezones are not supported. Most images default to UTC, but it is best practice to ensure this is the case when considering how to define your budgets. -{{% /alert %}} - #### Duration Duration allows compound durations with minutes and hours values such as `10h5m` or `30m` or `160h`. Since cron syntax does not accept denominations smaller than minutes, users can only define minutes or hours. @@ -295,16 +315,15 @@ metadata: #### Example: Disable Disruption on a NodePool -NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in disruption actions. +To disable disruption for all nodes launched by a NodePool, you can configure its `.spec.disruption.budgets`. Setting a budget of zero nodes will prevent any of those nodes from being considered for voluntary disruption. ```yaml -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default spec: - template: - metadata: - annotations: # will be applied to all nodes - karpenter.sh/do-not-disrupt: "true" + disruption: + budgets: + - nodes: "0" ``` diff --git a/website/content/en/preview/concepts/nodeclaims.md b/website/content/en/preview/concepts/nodeclaims.md new file mode 100644 index 000000000000..325d98bd5124 --- /dev/null +++ b/website/content/en/preview/concepts/nodeclaims.md @@ -0,0 +1,357 @@ +--- +title: "NodeClaims" +linkTitle: "NodeClaims" +weight: 30 +description: > + Understand NodeClaims +--- + +Karpenter uses NodeClaims to manage the lifecycle of Kubernetes Nodes with the underlying cloud provider. +Karpenter will create and delete NodeClaims in response to the demands of Pods in the cluster. +It does this by evaluating the requirements of pending pods, finding a compatible [NodePool]({{< ref "./nodepools" >}}) and [NodeClass]({{< ref "./nodeclasses" >}}) pair, and creating a NodeClaim which meets both sets of requirements. +Although NodeClaims are immutable resources managed by Karpenter, you can monitor NodeClaims to keep track of the status of your Nodes. + +In addition to tracking the lifecycle of Nodes, NodeClaims serve as requests for capacity. +Karpenter creates NodeClaims in response to provisioning and disruption needs (pre-spin). Whenever Karpenter +creates a NodeClaim, it asks the cloud provider to create the instance (launch), register and link the created node +with the NodeClaim (registration), and wait for the node and its resources to be ready (initialization). + +This page describes how NodeClaims integrate throughout Karpenter and the cloud provider implementation. + +If you want to learn more about the nodes being managed by Karpenter, you can either look directly at the NodeClaim or at the nodes they are associated with: + +* Checking NodeClaims: +If something goes wrong in the process of creating a node, you can look at the NodeClaim +to see where the node creation process might have failed. `kubectl get nodeclaims` will show you the NodeClaims +for the cluster, and its linked node. Using `kubectl describe nodeclaim ` will show the status of a particular NodeClaim. +For example, if the node is NotReady, you might see statuses indicating that the NodeClaim failed to launch, register, or initialize. +There will be logs emitted by the Karpenter controller to indicate this too. + +* Checking nodes: +Use commands such as `kubectl get node` and `kubectl describe node ` to see the actual resources, +labels, and other attributes associated with a particular node. + +## NodeClaim roles in node creation + +NodeClaims provide a critical role in the Karpenter workflow for provisioning capacity, and in node disruptions. + +The following diagram illustrates how NodeClaims interact with other components during Karpenter-driven node creation. + +![nodeclaim-node-creation](/nodeclaims.png) + +{{% alert title="Note" color="primary" %}} +Configure the `KARPENTER_NAMESPACE` environment variable to the namespace where you've installed Karpenter (`kube-system` is the default). Follow along with the Karpenter logs in your cluster and do the following: + +```bash +export KARPENTER_NAMESPACE="kube-system" +kubectl logs -f -n "${KARPENTER_NAMESPACE}" \ + -l app.kubernetes.io/name=karpenter +``` +In a separate terminal, start some pods that would require Karpenter to create nodes to handle those pods. +For example, start up some inflate pods as described in [Scale up deployment]({{< ref "../getting-started/getting-started-with-karpenter/#6-scale-up-deployment" >}}). +{{% /alert %}} + +As illustrated in the previous diagram, Karpenter interacts with NodeClaims and related components when creating a node: + +1. Watches for pods and monitors NodePools and NodeClasses: + * Checks the pod scheduling constraints and resource requests. + * Cross-references the requirements with the existing NodePools and NodeClasses, (e.g. zones, arch, os) + + **Example log:** + ```json + { + "level": "INFO", + "time": "2024-06-22T02:24:16.114Z", + "message": "found provisionable pod(s)", + "commit": "490ef94", + "Pods": "default/inflate-66fb68585c-xvs86, default/inflate-66fb68585c-hpcdz, default/inflate-66fb68585c-8xztf,01234567adb205c7e default/inflate-66fb68585c-t29d8, default/inflate-66fb68585c-nxflz", + "duration": "100.761702ms" + } + ``` + +2. Computes the shape and size of a NodeClaim (or NodeClaims) to create in the cluster to fit the set of pods from step 1. + + **Example log:** + ```json + { + "level": "INFO", + "time": "2024-06-22T02:24:16.114Z", + "message": "computed new nodeclaim(s) to fit pod(s)", + "controller": "provisioner", + "nodeclaims": 1, + "pods": 5 + } + ``` + +3. Creates the NodeClaim object in the cluster. + + **Example log:** + ```json + { + "level": "INFO", + "time": "2024-06-22T02:24:16.128Z", + "message": "created nodeclaim", + "controller": "provisioner", + "NodePool": { + "name":"default" + }, + "NodeClaim": { + "name":"default-sfpsl" + }, + "requests": { + "cpu":"5150m", + "pods":"8" + }, + "instance-types": "c3.2xlarge, c4.2xlarge, c4.4xlarge, c5.2xlarge, c5.4xlarge and 55 other(s)" + } + ``` + +4. Finds the new NodeClaim and translates it into an API call to create a cloud provider instance, logging + the response of the API call. + + If the API response is an unrecoverable error, such as an Insufficient Capacity Error, Karpenter will delete the NodeClaim, mark that instance type as temporarily unavailable, and create another NodeClaim if necessary. + + **Example log:** + ```json + { + "level": "INFO", + "time": "2024-06-22T02:24:19.028Z", + "message": "launched nodeclaim", + "controller": "nodeclaim.lifecycle", + "NodeClaim": { + "name": "default-sfpsl" + }, + "provider-id": "aws:///us-west-2b/i-01234567adb205c7e", + "instance-type": "c3.2xlarge", + "zone": "us-west-2b", + "capacity-type": "spot", + "allocatable": { + "cpu": "7910m", + "ephemeral-storage": "17Gi", + "memory": "13215Mi", + "pods": "58" + } + } + ``` + +5. Karpenter watches for the instance to register itself with the cluster as a node, and updates the node's + labels, annotations, taints, owner refs, and finalizer to match what was defined in the NodePool and NodeClaim. Once this step is + completed, Karpenter will remove the `karpenter.sh/unregistered` taint from the Node. + + If this fails to succeed within 15 minutes, Karpenter will remove the NodeClaim from the cluster and delete + the underlying instance, creating another NodeClaim if necessary. + + **Example log:** + ```json + { + "level": "INFO", + "time": "2024-06-22T02:26:19.028Z", + "message": "registered nodeclaim", + "controller": "nodeclaim.lifecycle", + "NodeClaim": { + "name": "default-sfpsl" + }, + "provider-id": "aws:///us-west-2b/i-01234567adb205c7e", + "Node": { + "name": "ip-xxx-xxx-xx-xxx.us-west-2.compute.internal" + } + } + ``` + +6. Karpenter continues to watch the node, waiting until the node becomes ready, has all its startup taints removed, + and has all requested resources registered on the node. + + **Example log:** + ```json + { + "level": "INFO", + "time": "2024-06-22T02:24:52.642Z", + "message": "initialized nodeclaim", + "controller": "nodeclaim.lifecycle", + "NodeClaim": { + "name": "default-sfpsl" + }, + "provider-id": "aws:///us-west-2b/i-01234567adb205c7e", + "Node": { + "name": "ip-xxx-xxx-xx-xxx.us-west-2.compute.internal" + }, + "allocatable": { + "cpu": "7910m", + "ephemeral-storage": "18242267924", + "hugepages-2Mi": "0", + "memory": "14320468Ki", + "pods": "58" + } + } + ``` + +## NodeClaim example +The following is an example of a NodeClaim. Keep in mind that you cannot modify a NodeClaim. +To see the contents of a NodeClaim, get the name of your NodeClaim, then run `kubectl describe` to see its contents: + +``` +kubectl get nodeclaim +NAME TYPE ZONE NODE READY AGE +default-m6pzn c7i-flex.2xlarge us-west-1a ip-xxx-xxx-xx-xxx.us-west-1.compute.internal True 7m50s + +kubectl describe nodeclaim default-m6pzn +``` +Starting at the bottom of this example, here are some highlights of what the NodeClaim contains: + +* The Node Name (ip-xxx-xxx-xx-xxx.us-west-1.compute.internal) and Provider ID (aws:///us-west-1a/i-xxxxxxxxxxxxxxxxx) identify the instance that is fulfilling this NodeClaim. +* Image ID (ami-0ccbbed159cce4e37) represents the operating system image running on the node. +* Status shows the resources that are available on the node (CPU, memory, and so on) as well as the conditions associated with the node. The conditions show the status of the node, including whether the node is launched, registered, and initialized. This is particularly useful if Pods are not deploying to the node and you want to determine the cause. +* Spec contains the metadata required for Karpenter to launch and manage an instance. This includes any scheduling requirements, resource requirements, the NodeClass reference, taints, and immutable disruption fields (expireAfter and terminationGracePeriod). +* Additional information includes annotations and labels which should be synced to the Node, creation metadata, the termination finalizer, and the owner reference. + +``` +Name: default-x9wxq +Namespace: +Labels: karpenter.k8s.aws/instance-category=c + karpenter.k8s.aws/instance-cpu=8 + karpenter.k8s.aws/instance-cpu-manufacturer=amd + karpenter.k8s.aws/instance-ebs-bandwidth=3170 + karpenter.k8s.aws/instance-encryption-in-transit-supported=true + karpenter.k8s.aws/instance-family=c5a + karpenter.k8s.aws/instance-generation=5 + karpenter.k8s.aws/instance-hypervisor=nitro + karpenter.k8s.aws/instance-memory=16384 + karpenter.k8s.aws/instance-network-bandwidth=2500 + karpenter.k8s.aws/instance-size=2xlarge + karpenter.sh/capacity-type=spot + karpenter.sh/nodepool=default + kubernetes.io/arch=amd64 + kubernetes.io/os=linux + node.kubernetes.io/instance-type=c5a.2xlarge + topology.k8s.aws/zone-id=usw2-az3 + topology.kubernetes.io/region=us-west-2 + topology.kubernetes.io/zone=us-west-2c +Annotations: compatibility.karpenter.k8s.aws/cluster-name-tagged: true + compatibility.karpenter.k8s.aws/kubelet-drift-hash: 15379597991425564585 + karpenter.k8s.aws/ec2nodeclass-hash: 5763643673275251833 + karpenter.k8s.aws/ec2nodeclass-hash-version: v3 + karpenter.k8s.aws/tagged: true + karpenter.sh/nodepool-hash: 377058807571762610 + karpenter.sh/nodepool-hash-version: v3 +API Version: karpenter.sh/v1 +Kind: NodeClaim +Metadata: + Creation Timestamp: 2024-08-07T05:37:30Z + Finalizers: + karpenter.sh/termination + Generate Name: default- + Generation: 1 + Owner References: + API Version: karpenter.sh/v1 + Block Owner Deletion: true + Kind: NodePool + Name: default + UID: 6b9c6781-ac05-4a4c-ad6a-7551a07b2ce7 + Resource Version: 19600526 + UID: 98a2ba32-232d-45c4-b7c0-b183cfb13d93 +Spec: + Expire After: 720h0m0s + Node Class Ref: + Group: + Kind: EC2NodeClass + Name: default + Requirements: + Key: kubernetes.io/arch + Operator: In + Values: + amd64 + Key: kubernetes.io/os + Operator: In + Values: + linux + Key: karpenter.sh/capacity-type + Operator: In + Values: + spot + Key: karpenter.k8s.aws/instance-category + Operator: In + Values: + c + m + r + Key: karpenter.k8s.aws/instance-generation + Operator: Gt + Values: + 2 + Key: karpenter.sh/nodepool + Operator: In + Values: + default + Key: node.kubernetes.io/instance-type + Operator: In + Values: + c3.xlarge + c4.xlarge + c5.2xlarge + c5.xlarge + c5a.xlarge + c5ad.2xlarge + c5ad.xlarge + c5d.2xlarge + Resources: + Requests: + Cpu: 3150m + Pods: 6 + Startup Taints: + Effect: NoSchedule + Key: app.dev/example-startup + Taints: + Effect: NoSchedule + Key: app.dev/example + Termination Grace Period: 1h0m0s +Status: + Allocatable: + Cpu: 7910m + Ephemeral - Storage: 17Gi + Memory: 14162Mi + Pods: 58 + vpc.amazonaws.com/pod-eni: 38 + Capacity: + Cpu: 8 + Ephemeral - Storage: 20Gi + Memory: 15155Mi + Pods: 58 + vpc.amazonaws.com/pod-eni: 38 + Conditions: + Last Transition Time: 2024-08-07T05:38:08Z + Message: + Reason: Consolidatable + Status: True + Type: Consolidatable + Last Transition Time: 2024-08-07T05:38:07Z + Message: + Reason: Initialized + Status: True + Type: Initialized + Last Transition Time: 2024-08-07T05:37:33Z + Message: + Reason: Launched + Status: True + Type: Launched + Last Transition Time: 2024-08-07T05:38:07Z + Message: + Reason: Ready + Status: True + Type: Ready + Last Transition Time: 2024-08-07T05:37:55Z + Message: + Reason: Registered + Status: True + Type: Registered + Image ID: ami-08946d4d49fc3f27b + Node Name: ip-xxx-xxx-xxx-xxx.us-west-2.compute.internal + Provider ID: aws:///us-west-2c/i-01234567890123 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Launched 70s karpenter Status condition transitioned, Type: Launched, Status: Unknown -> True, Reason: Launched + Normal DisruptionBlocked 70s karpenter Cannot disrupt NodeClaim: state node doesn't contain both a node and a nodeclaim + Normal Registered 48s karpenter Status condition transitioned, Type: Registered, Status: Unknown -> True, Reason: Registered + Normal Initialized 36s karpenter Status condition transitioned, Type: Initialized, Status: Unknown -> True, Reason: Initialized + Normal Ready 36s karpenter Status condition transitioned, Type: Ready, Status: Unknown -> True, Reason: Ready +``` diff --git a/website/content/en/preview/concepts/nodeclasses.md b/website/content/en/preview/concepts/nodeclasses.md index d426f579e7e6..13df18921c90 100644 --- a/website/content/en/preview/concepts/nodeclasses.md +++ b/website/content/en/preview/concepts/nodeclasses.md @@ -1,4 +1,4 @@ - --- +--- title: "NodeClasses" linkTitle: "NodeClasses" weight: 2 @@ -11,7 +11,7 @@ Each NodePool must reference an EC2NodeClass using `spec.template.spec.nodeClass Multiple NodePools may point to the same EC2NodeClass. ```yaml -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -19,15 +19,43 @@ spec: template: spec: nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass name: default --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: default spec: + kubelet: + podsPerCore: 2 + maxPods: 20 + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + kubeReserved: + cpu: 200m + memory: 100Mi + ephemeral-storage: 3Gi + evictionHard: + memory.available: 5% + nodefs.available: 10% + nodefs.inodesFree: 10% + evictionSoft: + memory.available: 500Mi + nodefs.available: 15% + nodefs.inodesFree: 15% + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + evictionMaxPodGracePeriod: 60 + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + cpuCFSQuota: true + clusterDNS: ["10.0.1.100"] # Required, resolves a default ami and userdata amiFamily: AL2 @@ -66,26 +94,19 @@ spec: # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes instanceProfile: "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" - # Optional, discovers amis to override the amiFamily's default amis # Each term in the array of amiSelectorTerms is ORed together # Within a single term, all conditions are ANDed amiSelectorTerms: # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" + - alias: al2023@v20240625 # Use alias to select a particular EKS optimized AMI - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" environment: test - name: my-ami - id: ami-123 - # Optional, use instance-store volumes for node ephemeral-storage - instanceStorePolicy: RAID0 - - # Optional, overrides autogenerated userdata with a merge semantic - userData: | - echo "Hello world" - # Optional, propagates tags to underlying EC2 resources tags: team: team-a @@ -95,7 +116,7 @@ spec: metadataOptions: httpEndpoint: enabled httpProtocolIPv6: disabled - httpPutResponseHopLimit: 2 + httpPutResponseHopLimit: 1 # This is changed to disable IMDS access from containers not on the host network httpTokens: required # Optional, configures storage devices for the instance @@ -111,6 +132,13 @@ spec: throughput: 125 snapshotID: snap-0123456789 + # Optional, use instance-store volumes for node ephemeral-storage + instanceStorePolicy: RAID0 + + # Optional, overrides autogenerated userdata with a merge semantic + userData: | + echo "Hello world" + # Optional, configures detailed monitoring for the instance detailedMonitoring: true @@ -159,12 +187,209 @@ status: # Generated instance profile name from "role" instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" + conditions: + - lastTransitionTime: "2024-02-02T19:54:34Z" + status: "True" + type: InstanceProfileReady + - lastTransitionTime: "2024-02-02T19:54:34Z" + status: "True" + type: SubnetsReady + - lastTransitionTime: "2024-02-02T19:54:34Z" + status: "True" + type: SecurityGroupsReady + - lastTransitionTime: "2024-02-02T19:54:34Z" + status: "True" + type: AMIsReady + - lastTransitionTime: "2024-02-02T19:54:34Z" + status: "True" + type: Ready +``` +Refer to the [NodePool docs]({{}}) for settings applicable to all providers. To explore various `EC2NodeClass` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/main/examples/v1/). + + +## spec.kubelet + +Karpenter provides the ability to specify a few additional Kubelet args. These are all optional and provide support for +additional customization and use cases. Adjust these only if you know you need to do so. For more details on kubelet configuration arguments, [see the KubeletConfiguration API specification docs](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1/). +The implemented fields are a subset of the full list of upstream kubelet configuration arguments. Please cut an issue if you'd like to see another field implemented. + +```yaml +kubelet: + podsPerCore: 2 + maxPods: 20 + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + kubeReserved: + cpu: 200m + memory: 100Mi + ephemeral-storage: 3Gi + evictionHard: + memory.available: 5% + nodefs.available: 10% + nodefs.inodesFree: 10% + evictionSoft: + memory.available: 500Mi + nodefs.available: 15% + nodefs.inodesFree: 15% + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + evictionMaxPodGracePeriod: 60 + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + cpuCFSQuota: true + clusterDNS: ["10.0.1.100"] +``` + +#### Pods Per Core + +An alternative way to dynamically set the maximum density of pods on a node is to use the `.spec.kubelet.podsPerCore` value. Karpenter will calculate the pod density during scheduling by multiplying this value by the number of logical cores (vCPUs) on an instance type. This value will also be passed through to the `--pods-per-core` value on kubelet startup to configure the number of allocatable pods the kubelet can assign to the node instance. + +The value generated from `podsPerCore` cannot exceed `maxPods`, meaning, if both are set, the minimum of the `podsPerCore` dynamic pod density and the static `maxPods` value will be used for scheduling. + +{{% alert title="Note" color="primary" %}} +`maxPods` may not be set in the `kubelet` of an EC2NodeClass, but may still be restricted by the `ENI_LIMITED_POD_DENSITY` value. You may want to ensure that the `podsPerCore` value that will be used for instance families associated with the EC2NodeClass will not cause unexpected behavior by exceeding the `maxPods` value. +{{% /alert %}} + +{{% alert title="Pods Per Core on Bottlerocket" color="warning" %}} +Bottlerocket AMIFamily currently does not support `podsPerCore` configuration. If a EC2NodeClass contains a `provider` or `providerRef` to a node template that will launch a Bottlerocket instance, the `podsPerCore` value will be ignored for scheduling and for configuring the kubelet. +{{% /alert %}} + +#### Max Pods + +For small instances that require an increased pod density or large instances that require a reduced pod density, you can override this default value with `.spec.kubelet.maxPods`. This value will be used during Karpenter pod scheduling and passed through to `--max-pods` on kubelet startup. + +{{% alert title="Note" color="primary" %}} +When using small instance types, it may be necessary to enable [prefix assignment mode](https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/) in the AWS VPC CNI plugin to support a higher pod density per node. Prefix assignment mode was introduced in AWS VPC CNI v1.9 and allows ENIs to manage a broader set of IP addresses. Much higher pod densities are supported as a result. +{{% /alert %}} + +{{% alert title="Windows Support Notice" color="warning" %}} +Presently, Windows worker nodes do not support using more than one ENI. +As a consequence, the number of IP addresses, and subsequently, the number of pods that a Windows worker node can support is limited by the number of IPv4 addresses available on the primary ENI. +Currently, Karpenter will only consider individual secondary IP addresses when calculating the pod density limit. +{{% /alert %}} + +### Reserved Resources + +Karpenter will automatically configure the system and kube reserved resource requests on the fly on your behalf. These requests are used to configure your node and to make scheduling decisions for your pods. If you have specific requirements or know that you will have additional capacity requirements, you can optionally override the `--system-reserved` configuration defaults with the `.spec.kubelet.systemReserved` values and the `--kube-reserved` configuration defaults with the `.spec.kubelet.kubeReserved` values. + +{{% alert title="Note" color="primary" %}} +Karpenter considers these reserved resources when computing the allocatable ephemeral storage on a given instance type. +If `kubeReserved` is not specified, Karpenter will compute the default reserved [CPU](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L251) and [memory](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L235) resources for the purpose of ephemeral storage computation. +These defaults are based on the defaults on Karpenter's supported AMI families, which are not the same as the kubelet defaults. +You should be aware of the CPU and memory default calculation when using Custom AMI Families. If they don't align, there may be a difference in Karpenter's computed allocatable ephemeral storage and the actually ephemeral storage available on the node. +{{% /alert %}} + +### Eviction Thresholds + +The kubelet supports eviction thresholds by default. When enough memory or file system pressure is exerted on the node, the kubelet will begin to evict pods to ensure that system daemons and other system processes can continue to run in a healthy manner. + +Kubelet has the notion of [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds). In hard evictions, pods are evicted as soon as a threshold is met, with no grace period to terminate. Soft evictions, on the other hand, provide an opportunity for pods to be terminated gracefully. They do so by sending a termination signal to pods that are planning to be evicted and allowing those pods to terminate up to their grace period. + +Karpenter supports [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) through the `.spec.kubelet.evictionHard` field and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds) through the `.spec.kubelet.evictionSoft` field. `evictionHard` and `evictionSoft` are configured by listing [signal names](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#eviction-signals) with either percentage values or resource values. + +```yaml +kubelet: + evictionHard: + memory.available: 500Mi + nodefs.available: 10% + nodefs.inodesFree: 10% + imagefs.available: 5% + imagefs.inodesFree: 5% + pid.available: 7% + evictionSoft: + memory.available: 1Gi + nodefs.available: 15% + nodefs.inodesFree: 15% + imagefs.available: 10% + imagefs.inodesFree: 10% + pid.available: 10% +``` + +#### Supported Eviction Signals + +| Eviction Signal | Description | +|--------------------|---------------------------------------------------------------------------------| +| memory.available | memory.available := node.status.capacity[memory] - node.stats.memory.workingSet | +| nodefs.available | nodefs.available := node.stats.fs.available | +| nodefs.inodesFree | nodefs.inodesFree := node.stats.fs.inodesFree | +| imagefs.available | imagefs.available := node.stats.runtime.imagefs.available | +| imagefs.inodesFree | imagefs.inodesFree := node.stats.runtime.imagefs.inodesFree | +| pid.available | pid.available := node.stats.rlimit.maxpid - node.stats.rlimit.curproc | + +For more information on eviction thresholds, view the [Node-pressure Eviction](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction) section of the official Kubernetes docs. + +#### Soft Eviction Grace Periods + +Soft eviction pairs an eviction threshold with a specified grace period. With soft eviction thresholds, the kubelet will only begin evicting pods when the node exceeds its soft eviction threshold over the entire duration of its grace period. For example, if you specify `evictionSoft[memory.available]` of `500Mi` and a `evictionSoftGracePeriod[memory.available]` of `1m30`, the node must have less than `500Mi` of available memory over a minute and a half in order for the kubelet to begin evicting pods. + +Optionally, you can specify an `evictionMaxPodGracePeriod` which defines the administrator-specified maximum pod termination grace period to use during soft eviction. If a namespace-owner had specified a pod `terminationGracePeriodInSeconds` on pods in their namespace, the minimum of `evictionPodGracePeriod` and `terminationGracePeriodInSeconds` would be used. + +```yaml +kubelet: + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + imagefs.available: 1m30s + imagefs.inodesFree: 2m + pid.available: 2m + evictionMaxPodGracePeriod: 60 ``` -Refer to the [NodePool docs]({{}}) for settings applicable to all providers. To explore various `EC2NodeClass` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). + +### Pod Density + +By default, the number of pods on a node is limited by both the number of networking interfaces (ENIs) that may be attached to an instance type and the number of IP addresses that can be assigned to each ENI. See [IP addresses per network interface per instance type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI) for a more detailed information on these instance types' limits. + +{{% alert title="Note" color="primary" %}} +By default, the VPC CNI allocates IPs for a node and pods from the same subnet. With [VPC CNI Custom Networking](https://aws.github.io/aws-eks-best-practices/networking/custom-networking), the pods will receive IP addresses from another subnet dedicated to pod IPs. This approach makes it easier to manage IP addresses and allows for separate Network Access Control Lists (NACLs) applied to your pods. VPC CNI Custom Networking reduces the pod density of a node since one of the ENI attachments will be used for the node and cannot share the allocated IPs on the interface to pods. Karpenter supports VPC CNI Custom Networking and similar CNI setups where the primary node interface is separated from the pods interfaces through a global [setting](./settings.md#configmap) within the karpenter-global-settings configmap: `aws.reservedENIs`. In the common case, `aws.reservedENIs` should be set to `"1"` if using Custom Networking. +{{% /alert %}} + +{{% alert title="Windows Support Notice" color="warning" %}} +It's currently not possible to specify custom networking with Windows nodes. +{{% /alert %}} ## spec.amiFamily -AMIFamily is a required field, dictating both the default bootstrapping logic for nodes provisioned through this `EC2NodeClass` but also selecting a group of recommended, latest AMIs by default. Currently, Karpenter supports `amiFamily` values `AL2`, `AL2023`, `Bottlerocket`, `Ubuntu`, `Windows2019`, `Windows2022` and `Custom`. GPUs are only supported by default with `AL2` and `Bottlerocket`. The `AL2` amiFamily does not support ARM64 GPU instance types unless you specify custom [`amiSelectorTerms`]({{}}). Default bootstrapping logic is shown below for each of the supported families. +AMIFamily dictates the default bootstrapping logic for nodes provisioned through this `EC2NodeClass`. +An `amiFamily` is only required if you don't specify a `spec.amiSelectorTerms.alias` object. +For example, if you specify `alias: al2023@v20240625`, the `amiFamily` is implicitly `AL2023`. + +AMIFamily does not impact which AMI is discovered, only the UserData generation and default BlockDeviceMappings. To automatically discover EKS optimized AMIs, use the new [`alias` field in amiSelectorTerms]({{< ref "#specamiselectorterms" >}}). + + +{{% alert title="Ubuntu Support Dropped at v1" color="warning" %}} + +Support for the Ubuntu AMIFamily has been dropped at Karpenter `v1.0.0`. +This means Karpenter no longer supports automatic AMI discovery and UserData generation for Ubuntu. +To continue using Ubuntu AMIs, you will need to select Ubuntu AMIs using `amiSelectorTerms`. + +Additionally, you will need to either maintain UserData yourself using the `Custom` AMIFamily, or you can use the `AL2` AMIFamily and custom `blockDeviceMappings` (as shown below). +The `AL2` family has an identical UserData format, but this compatibility isn't guaranteed long term. +Changes to AL2's or Ubuntu's UserData format could result in incompatibility, at which point the `Custom` AMIFamily must be used. + +**Ubuntu NodeClass Example:** +```yaml +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass +spec: + amiFamily: AL2 + amiSelectorTerms: + - id: ami-placeholder + blockDeviceMappings: + - deviceName: '/dev/sda1' + rootVolume: true + ebs: + encrypted: true + volumeType: gp3 + volumeSize: 20Gi +``` + +{{% /alert %}} + ### AL2 @@ -228,24 +453,6 @@ max-pods = 110 'karpenter.sh/nodepool' = 'test' ``` -### Ubuntu - -```bash -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="//" - ---// -Content-Type: text/x-shellscript; charset="us-ascii" - -#!/bin/bash -xe -exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 -/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ ---dns-cluster-ip '10.100.0.10' \ ---use-max-pods false \ ---kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test" --max-pods=110' ---//-- -``` - ### Windows2019 ```powershell @@ -270,7 +477,7 @@ Karpenter will automatically query for the appropriate [EKS optimized AMI](https ### Custom -The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families. +The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families. For this AMIFamily, kubelet must add the taint `karpenter.sh/unregistered:NoExecute` via the `--register-with-taints` flag ([flags](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#options)) or the KubeletConfiguration spec ([options](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1/#kubelet-config-k8s-io-v1-CredentialProviderConfig) and [docs](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)). Karpenter will fail to register nodes that do not have this taint. ## spec.subnetSelectorTerms @@ -427,17 +634,40 @@ spec: - id: "sg-06e0cf9c198874591" ``` +## spec.role + +`Role` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If using the [Karpenter Getting Started Guide]({{}}) to deploy Karpenter, you can use the `KarpenterNodeRole-$CLUSTER_NAME` role provisioned by that process. + +```yaml +spec: + role: "KarpenterNodeRole-$CLUSTER_NAME" +``` + +## spec.instanceProfile + +`InstanceProfile` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf; instead, it expects that you have pre-provisioned an IAM instance profile and assigned it a role. + +You can provision and assign a role to an IAM instance profile using [CloudFormation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-instanceprofile.html) or by using the [`aws iam create-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/create-instance-profile.html) and [`aws iam add-role-to-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/add-role-to-instance-profile.html) commands in the CLI. + +{{% alert title="Note" color="primary" %}} + +For [private clusters](https://docs.aws.amazon.com/eks/latest/userguide/private-clusters.html) that do not have access to the public internet, using `spec.instanceProfile` is required. `spec.role` cannot be used since Karpenter needs to access IAM endpoints to manage a generated instance profile. IAM [doesn't support private endpoints](https://docs.aws.amazon.com/vpc/latest/privatelink/aws-services-privatelink-support.html) to enable accessing the service without going to the public internet. + +{{% /alert %}} + ## spec.amiSelectorTerms -AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). **When you specify `amiSelectorTerms`, you fully override the default AMIs that are selected on by your EC2NodeClass [`amiFamily`]({{< ref "#specamifamily" >}}).** +AMI Selector Terms are __required__ and are used to configure AMIs for Karpenter to use. AMIs are discovered through alias, id, owner, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. ```yaml amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # Select on any AMI that has an al2023 AMI family and 20240625 version, + # and both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" + - alias: al2023@v20240625 - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" environment: test @@ -445,7 +675,29 @@ amiSelectorTerms: - id: ami-123 ``` -This field is optional, and Karpenter will use the latest EKS-optimized AMIs for the AMIFamily if no amiSelectorTerms are specified. To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To ensure that AMIs are owned by the expected owner, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. +An `alias` has the following format: `family@version`. +Use the `alias` field to select an EKS-optimized AMI family and version. Family can be one of the following values: + +* `al2` +* `al2023` +* `bottlerocket` +* `windows2019` +* `windows2022` + +The version string can be set to `latest`, or pinned to a specific AMI using the format of that AMI's GitHub release tags. +For example, AL2 and AL2023 use dates for their release, so they can be pinned as follows: +``` +alias: al2023@v20240703 +``` +Bottlerocket uses a semantic version for their releases. You can pin bottlerocket as follows: +``` +alias: bottlerocket@v1.20.4 +``` +The Windows family does not support pinning, so only `latest` is supported. + +An `alias` is mutually exclusive and may not be specified with any other terms. + +To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To select AMIs that are not owned by `amazon` or the account that Karpenter is running in, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. If owner is not set for `name`, it defaults to `self,amazon`, preventing Karpenter from inadvertently selecting an AMI that is owned by a different account. Tags don't require an owner as tags can only be discovered by the user who created them. @@ -464,7 +716,14 @@ If `amiSelectorTerms` match more than one AMI, Karpenter will automatically dete #### Examples +Select by AMI family and version: +```yaml + amiSelectorTerms: + - alias: al2023@v20240625 +``` + Select all with a specified tag: + ```yaml amiSelectorTerms: - tags: @@ -515,27 +774,6 @@ Specify using ids: - id: "ami-456" ``` -## spec.role - -`Role` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If using the [Karpenter Getting Started Guide]({{}}) to deploy Karpenter, you can use the `KarpenterNodeRole-$CLUSTER_NAME` role provisioned by that process. - -```yaml -spec: - role: "KarpenterNodeRole-$CLUSTER_NAME" -``` - -## spec.instanceProfile - -`InstanceProfile` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf; instead, it expects that you have pre-provisioned an IAM instance profile and assigned it a role. - -You can provision and assign a role to an IAM instance profile using [CloudFormation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-instanceprofile.html) or by using the [`aws iam create-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/create-instance-profile.html) and [`aws iam add-role-to-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/add-role-to-instance-profile.html) commands in the CLI. - -{{% alert title="Note" color="primary" %}} - -For [private clusters](https://docs.aws.amazon.com/eks/latest/userguide/private-clusters.html) that do not have access to the public internet, using `spec.instanceProfile` is required. `spec.role` cannot be used since Karpenter needs to access IAM endpoints to manage a generated instance profile. IAM [doesn't support private endpoints](https://docs.aws.amazon.com/vpc/latest/privatelink/aws-services-privatelink-support.html) to enable accessing the service without going to the public internet. - -{{% /alert %}} - ## spec.tags Karpenter adds tags to all resources it creates, including EC2 Instances, EBS volumes, and Launch Templates. The default set of tags are listed below. @@ -546,6 +784,7 @@ karpenter.sh/nodeclaim: karpenter.sh/nodepool: karpenter.k8s.aws/ec2nodeclass: kubernetes.io/cluster/: owned +eks:eks-cluster-name: ``` Additional tags can be added in the tags section, which will be merged with the default tags specified above. @@ -574,7 +813,7 @@ spec: metadataOptions: httpEndpoint: enabled httpProtocolIPv6: disabled - httpPutResponseHopLimit: 2 + httpPutResponseHopLimit: 1 httpTokens: required ``` @@ -639,17 +878,6 @@ spec: encrypted: true ``` -### Ubuntu -```yaml -spec: - blockDeviceMappings: - - deviceName: /dev/sda1 - ebs: - volumeSize: 20Gi - volumeType: gp3 - encrypted: true -``` - ### Windows2019/Windows2022 ```yaml spec: @@ -703,7 +931,7 @@ Since the Kubelet & Containerd will be using the instance-store filesystem, you You can control the UserData that is applied to your worker nodes via this field. This allows you to run custom scripts or pass-through custom configuration to Karpenter instances on start-up. ```yaml -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: bottlerocket-example @@ -727,7 +955,7 @@ See [Node NotReady]({{< ref "../troubleshooting/#node-notready" >}}) troubleshoo {{% /alert %}} ```yaml -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: al2-example @@ -745,11 +973,11 @@ spec: chown -R ec2-user ~ec2-user/.ssh ``` -For more examples on configuring fields for different AMI families, see the [examples here](https://github.com/aws/karpenter/blob/main/examples/v1beta1). +For more examples on configuring fields for different AMI families, see the [examples here](https://github.com/aws/karpenter/blob/main/examples/v1). Karpenter will merge the userData you specify with the default userData for that AMIFamily. See the [AMIFamily]({{< ref "#specamifamily" >}}) section for more details on these defaults. View the sections below to understand the different merge strategies for each AMIFamily. -### AL2/Ubuntu +### AL2 * Your UserData can be in the [MIME multi part archive](https://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive) format. * Karpenter will transform your custom user-data as a MIME part, if necessary, and then merge a final MIME part to the end of your UserData parts which will bootstrap the worker node. Karpenter will have full control over all the parameters being passed to the bootstrap script. @@ -829,7 +1057,7 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 You can also set kubelet-config properties by modifying the kubelet-config.json file before the EKS bootstrap script starts the kubelet: ```yaml -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: kubelet-config-example @@ -846,7 +1074,12 @@ spec: * Your UserData may be in one of three formats: a [MIME multi part archive](https://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive), a NodeConfig YAML / JSON string, or a shell script. * Karpenter will transform your custom UserData into a MIME part, if necessary, and then create a MIME multi-part archive. This archive will consist of a generated NodeConfig, containing Karpenter's default values, followed by the transformed custom UserData. For more information on the NodeConfig spec, refer to the [AL2023 EKS Optimized AMI docs](https://awslabs.github.io/amazon-eks-ami/nodeadm/doc/examples/). -* If a value is specified both in the Karpenter generated NodeConfig and the same value is specified in the custom user data, the value in the custom user data will take precedence. + +{{% alert title="Warning" color="warning" %}} +Any values configured by the Karpenter generated NodeConfig object will take precedent over values specifed in `spec.userData`. +This includes cluster name, cluster CIDR, cluster endpoint, certificate authority, taints, labels, and any value in [spec.kubelet]({{< ref "#speckubelet" >}}). +These fields must be configured natively through Karpenter rather than through UserData. +{{% /alert %}} #### Passed-in UserData (NodeConfig) @@ -866,7 +1099,16 @@ MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="//" --// -# Karpenter Generated NodeConfig +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 + +--// Content-Type: application/node.eks.aws # Karpenter Generated NodeConfig @@ -886,15 +1128,6 @@ spec: flags: - --node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=default" ---// -Content-Type: application/node.eks.aws - -apiVersion: node.eks.aws/v1alpha1 -kind: NodeConfig -spec: - kubelet: - config: - maxPods: 42 --//-- ``` @@ -911,6 +1144,12 @@ echo "Hello, AL2023!" MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="//" +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" + --// Content-Type: application/node.eks.aws @@ -931,11 +1170,6 @@ spec: flags: - --node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=default" ---// -Content-Type: text/x-shellscript; charset="us-ascii" - -#!/bin/bash -echo "Hello, AL2023!" --//-- ``` @@ -945,6 +1179,12 @@ echo "Hello, AL2023!" MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="//" +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" + --// Content-Type: application/node.eks.aws @@ -955,11 +1195,6 @@ spec: config: maxPods: 42 --// -Content-Type: text/x-shellscript; charset="us-ascii" - -#!/bin/bash -echo "Hello, AL2023!" ---// ``` #### Merged UserData (MIME) @@ -971,6 +1206,21 @@ Content-Type: multipart/mixed; boundary="//" --// Content-Type: application/node.eks.aws +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" + +--// +Content-Type: application/node.eks.aws + # Karpenter Generated NodeConfig apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig @@ -988,32 +1238,20 @@ spec: flags: - --node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=default" ---// -Content-Type: application/node.eks.aws - -apiVersion: node.eks.aws/v1alpha1 -kind: NodeConfig -spec: - kubelet: - config: - maxPods: 42 ---// -Content-Type: text/x-shellscript; charset="us-ascii" - -#!/bin/bash -echo "Hello, AL2023!" --//-- ``` ### Bottlerocket * Your UserData must be valid TOML. -* Karpenter will automatically merge settings to ensure successful bootstrap including `cluster-name`, `api-server` and `cluster-certificate`. Any labels and taints that need to be set based on pod requirements will also be specified in the final merged UserData. - * All Kubelet settings that Karpenter applies will override the corresponding settings in the provided UserData. For example, if you've specified `settings.kubernetes.cluster-name`, it will be overridden. - * If MaxPods is specified via the binary arg to Karpenter, the value will override anything specified in the UserData. - * If ClusterDNS is specified via `spec.kubeletConfiguration`, then that value will override anything specified in the UserData. * Unknown TOML fields will be ignored when the final merged UserData is generated by Karpenter. +{{% alert title="Warning" color="warning" %}} +Any values configured by Karpenter will take precedent over values specifed in `spec.userData`. +This includes cluster name, cluster endpoint, cluster certificate, taints, labels, and any value in [spec.kubelet]({{< ref "#speckubelet" >}}). +These fields must be configured natively through Karpenter rather than through UserData. +{{% /alert %}} + Consider the following example to understand how your custom UserData settings will be merged in. #### Passed-in UserData @@ -1099,7 +1337,10 @@ spec: ## spec.associatePublicIPAddress -A boolean field that controls whether instances created by Karpenter for this EC2NodeClass will have an associated public IP address. This overrides the `MapPublicIpOnLaunch` setting applied to the subnet the node is launched in. If this field is not set, the `MapPublicIpOnLaunch` field will be respected. +You can explicitly set `AssociatePublicIPAddress: false` when you are only launching into private subnets. +Previously, Karpenter auto-set `associatePublicIPAddress` on the primary ENI to false if a user’s subnet options were all private subnets. +This value is a boolean field that controls whether instances created by Karpenter for this EC2NodeClass will have an associated public IP address. This overrides the `MapPublicIpOnLaunch` setting applied to the subnet the node is launched in. If this field is not set, the `MapPublicIpOnLaunch` field will be respected. + {{% alert title="Note" color="warning" %}} If a `NodeClaim` requests `vpc.amazonaws.com/efa` resources, `spec.associatePublicIPAddress` is respected. However, if this `NodeClaim` requests **multiple** EFA resources and the value for `spec.associatePublicIPAddress` is true, the instance will fail to launch. This is due to an EC2 restriction which @@ -1242,3 +1483,33 @@ spec: status: instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" ``` + +## status.conditions + +[`status.conditions`]({{< ref "#statusconditions" >}}) indicates EC2NodeClass readiness. This will be `Ready` when Karpenter successfully discovers AMIs, Instance Profile, Subnets, Cluster CIDR (AL2023 only) and SecurityGroups for the EC2NodeClass. + +```yaml +spec: + role: "KarpenterNodeRole-${CLUSTER_NAME}" +status: + conditions: + Last Transition Time: 2024-05-06T06:04:45Z + Message: Ready + Reason: Ready + Status: True + Type: Ready +``` + +If any of the underlying conditions are not resolved then `Status` is `False` and `Message` indicates the dependency that was not resolved. + +```yaml +spec: + role: "KarpenterNodeRole-${CLUSTER_NAME}" +status: + conditions: + Last Transition Time: 2024-05-06T06:19:46Z + Message: unable to resolve instance profile for node class + Reason: NodeClassNotReady + Status: False + Type: Ready +``` diff --git a/website/content/en/preview/concepts/nodepools.md b/website/content/en/preview/concepts/nodepools.md index d679bb5f4373..4610bac5afab 100644 --- a/website/content/en/preview/concepts/nodepools.md +++ b/website/content/en/preview/concepts/nodepools.md @@ -1,7 +1,7 @@ --- title: "NodePools" linkTitle: "NodePools" -weight: 1 +weight: 10 description: > Configure Karpenter with NodePools --- @@ -22,10 +22,15 @@ Here are things you should know about NodePools: * If Karpenter encounters a startup taint in the NodePool it will be applied to nodes that are provisioned, but pods do not need to tolerate the taint. Karpenter assumes that the taint is temporary and some other system will remove the taint. * It is recommended to create NodePools that are mutually exclusive. So no Pod should match multiple NodePools. If multiple NodePools are matched, Karpenter will use the NodePool with the highest [weight](#specweight). -For some example `NodePool` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). + +{{% alert title="Note" color="primary" %}} +Objects for setting Kubelet features have been moved from the NodePool spec to the EC2NodeClasses spec, to not require other Karpenter providers to support those features. +{{% /alert %}} + +For some example `NodePool` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/main/examples/v1/). ```yaml -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -46,7 +51,7 @@ spec: spec: # References the Cloud Provider's NodeClass resource, see your cloud provider specific documentation nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws # Updated since only a single version will be served kind: EC2NodeClass name: default @@ -63,6 +68,21 @@ spec: - key: example.com/another-taint effect: NoSchedule + # The amount of time a Node can live on the cluster before being removed + # Avoiding long-running Nodes helps to reduce security vulnerabilities as well as to reduce the chance of issues that can plague Nodes with long uptimes such as file fragmentation or memory leaks from system processes + # You can choose to disable expiration entirely by setting the string value 'Never' here + + # Note: changing this value in the nodepool will drift the nodeclaims. + expireAfter: 720h | Never + + # The amount of time that a node can be draining before it's forcibly deleted. A node begins draining when a delete call is made against it, starting + # its finalization flow. Pods with TerminationGracePeriodSeconds will be deleted preemptively before this terminationGracePeriod ends to give as much time to cleanup as possible. + # If your pod's terminationGracePeriodSeconds is larger than this terminationGracePeriod, Karpenter may forcibly delete the pod + # before it has its full terminationGracePeriod to cleanup. + + # Note: changing this value in the nodepool will drift the nodeclaims. + terminationGracePeriod: 48h + # Requirements that constrain the parameters of provisioned nodes. # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. # Operators { In, NotIn, Exists, DoesNotExist, Gt, and Lt } are supported. @@ -72,7 +92,7 @@ spec: operator: In values: ["c", "m", "r"] # minValues here enforces the scheduler to consider at least that number of unique instance-category to schedule the pods. - # This field is ALPHA and can be dropped or replaced at any time + # This field is ALPHA and can be dropped or replaced at any time minValues: 2 - key: "karpenter.k8s.aws/instance-family" operator: In @@ -97,55 +117,18 @@ spec: operator: In values: ["spot", "on-demand"] - # Karpenter provides the ability to specify a few additional Kubelet args. - # These are all optional and provide support for additional customization and use cases. - kubelet: - clusterDNS: ["10.0.1.100"] - systemReserved: - cpu: 100m - memory: 100Mi - ephemeral-storage: 1Gi - kubeReserved: - cpu: 200m - memory: 100Mi - ephemeral-storage: 3Gi - evictionHard: - memory.available: 5% - nodefs.available: 10% - nodefs.inodesFree: 10% - evictionSoft: - memory.available: 500Mi - nodefs.available: 15% - nodefs.inodesFree: 15% - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - evictionMaxPodGracePeriod: 60 - imageGCHighThresholdPercent: 85 - imageGCLowThresholdPercent: 80 - cpuCFSQuota: true - podsPerCore: 2 - maxPods: 20 - # Disruption section which describes the ways in which Karpenter can disrupt and replace Nodes # Configuration in this section constrains how aggressive Karpenter can be with performing operations # like rolling Nodes due to them hitting their maximum lifetime (expiry) or scaling down nodes to reduce cluster cost disruption: # Describes which types of Nodes Karpenter should consider for consolidation - # If using 'WhenUnderutilized', Karpenter will consider all nodes for consolidation and attempt to remove or replace Nodes when it discovers that the Node is underutilized and could be changed to reduce cost + # If using 'WhenEmptyOrUnderutilized', Karpenter will consider all nodes for consolidation and attempt to remove or replace Nodes when it discovers that the Node is empty or underutilized and could be changed to reduce cost # If using `WhenEmpty`, Karpenter will only consider nodes for consolidation that contain no workload pods - consolidationPolicy: WhenUnderutilized | WhenEmpty + consolidationPolicy: WhenEmptyOrUnderutilized | WhenEmpty - # The amount of time Karpenter should wait after discovering a consolidation decision - # This value can currently only be set when the consolidationPolicy is 'WhenEmpty' + # The amount of time Karpenter should wait to consolidate a node after a pod has been added or removed from the node. # You can choose to disable consolidation entirely by setting the string value 'Never' here - consolidateAfter: 30s - - # The amount of time a Node can live on the cluster before being removed - # Avoiding long-running Nodes helps to reduce security vulnerabilities as well as to reduce the chance of issues that can plague Nodes with long uptimes such as file fragmentation or memory leaks from system processes - # You can choose to disable expiration entirely by setting the string value 'Never' here - expireAfter: 720h + consolidateAfter: 1m | Never # Added to allow additional control over consolidation aggressiveness # Budgets control the speed Karpenter can scale down nodes. # Karpenter will respect the minimum of the currently active budgets, and will round up @@ -157,7 +140,7 @@ spec: duration: 8h nodes: "0" - # Resource limits constrain the total size of the cluster. + # Resource limits constrain the total size of the pool. # Limits prevent Karpenter from creating new instances once the limit is exceeded. limits: cpu: "1000" @@ -167,7 +150,49 @@ spec: # to select. Higher weights indicate higher priority when comparing NodePools. # Specifying no weight is equivalent to specifying a weight of 0. weight: 10 +status: + conditions: + - type: Initialized + status: "False" + observedGeneration: 1 + lastTransitionTime: "2024-02-02T19:54:34Z" + reason: NodeClaimNotLaunched + message: "NodeClaim hasn't succeeded launch" + resources: + cpu: "20" + memory: "8192Mi" + ephemeral-storage: "100Gi" ``` +## metadata.name +The name of the NodePool. + +## spec.template.metadata.labels +Arbitrary key/value pairs to apply to all nodes. + +## spec.template.metadata.annotations +Arbitrary key/value pairs to apply to all nodes. + +## spec.template.spec.nodeClassRef + +This field points to the Cloud Provider NodeClass resource. See [EC2NodeClasses]({{}}) for details. + +## spec.template.spec.taints + +Taints to add to provisioned nodes. Pods that don't tolerate those taints could be prevented from scheduling. +See [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for details. + +## spec.template.spec.startupTaints + +Taints that are added to nodes to indicate that a certain condition must be met, such as starting an agent or setting up networking, before the node is can be initialized. +These taints must be cleared before pods can be deployed to a node. + +## spec.template.spec.expireAfter + +The amount of time a Node can live on the cluster before being deleted by Karpenter. Nodes will begin draining once it's expiration has been hit. + +## spec.template.spec.terminationGracePeriod + +The amount of time a Node can be draining before Karpenter forcibly cleans up the node. Pods blocking eviction like PDBs and do-not-disrupt will be respected during draining until the `terminationGracePeriod` is reached, where those pods will be forcibly deleted. ## spec.template.spec.requirements @@ -233,6 +258,10 @@ Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand i Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. +{{% alert title="Note" color="primary" %}} +There is currently a limit of 100 on the total number of requirements on both the NodePool and the NodeClaim. It's important to note that `spec.template.metadata.labels` are also propagated as requirements on the NodeClaim when it's created, meaning that you can't have more than 100 requirements and labels combined set on your NodePool. +{{% /alert %}} + ### Min Values Along with the combination of [key,operator,values] in the requirements, Karpenter also supports `minValues` in the NodePool requirements block, allowing the scheduler to be aware of user-specified flexibility minimums while scheduling pods to a cluster. If Karpenter cannot meet this minimum flexibility for each key when scheduling a pod, it will fail the scheduling loop for that NodePool, either falling back to another NodePool which meets the pod requirements or failing scheduling the pod altogether. @@ -327,157 +356,12 @@ spec: {{% /alert %}} -## spec.template.spec.nodeClassRef - -This field points to the Cloud Provider NodeClass resource. Learn more about [EC2NodeClasses]({{}}). - -## spec.template.spec.kubelet - -Karpenter provides the ability to specify a few additional Kubelet args. These are all optional and provide support for -additional customization and use cases. Adjust these only if you know you need to do so. For more details on kubelet configuration arguments, [see the KubeletConfiguration API specification docs](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/). The implemented fields are a subset of the full list of upstream kubelet configuration arguments. Please cut an issue if you'd like to see another field implemented. - -```yaml -kubelet: - clusterDNS: ["10.0.1.100"] - systemReserved: - cpu: 100m - memory: 100Mi - ephemeral-storage: 1Gi - kubeReserved: - cpu: 200m - memory: 100Mi - ephemeral-storage: 3Gi - evictionHard: - memory.available: 5% - nodefs.available: 10% - nodefs.inodesFree: 10% - evictionSoft: - memory.available: 500Mi - nodefs.available: 15% - nodefs.inodesFree: 15% - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - evictionMaxPodGracePeriod: 60 - imageGCHighThresholdPercent: 85 - imageGCLowThresholdPercent: 80 - cpuCFSQuota: true - podsPerCore: 2 - maxPods: 20 -``` - -### Reserved Resources - -Karpenter will automatically configure the system and kube reserved resource requests on the fly on your behalf. These requests are used to configure your node and to make scheduling decisions for your pods. If you have specific requirements or know that you will have additional capacity requirements, you can optionally override the `--system-reserved` configuration defaults with the `.spec.template.spec.kubelet.systemReserved` values and the `--kube-reserved` configuration defaults with the `.spec.template.spec.kubelet.kubeReserved` values. - -{{% alert title="Note" color="primary" %}} -Karpenter considers these reserved resources when computing the allocatable ephemeral storage on a given instance type. -If `kubeReserved` is not specified, Karpenter will compute the default reserved [CPU](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L251) and [memory](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L235) resources for the purpose of ephemeral storage computation. -These defaults are based on the defaults on Karpenter's supported AMI families, which are not the same as the kubelet defaults. -You should be aware of the CPU and memory default calculation when using Custom AMI Families. If they don't align, there may be a difference in Karpenter's computed allocatable ephemeral storage and the actually ephemeral storage available on the node. -{{% /alert %}} - -### Eviction Thresholds - -The kubelet supports eviction thresholds by default. When enough memory or file system pressure is exerted on the node, the kubelet will begin to evict pods to ensure that system daemons and other system processes can continue to run in a healthy manner. - -Kubelet has the notion of [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds). In hard evictions, pods are evicted as soon as a threshold is met, with no grace period to terminate. Soft evictions, on the other hand, provide an opportunity for pods to be terminated gracefully. They do so by sending a termination signal to pods that are planning to be evicted and allowing those pods to terminate up to their grace period. - -Karpenter supports [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) through the `.spec.template.spec.kubelet.evictionHard` field and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds) through the `.spec.template.spec.kubelet.evictionSoft` field. `evictionHard` and `evictionSoft` are configured by listing [signal names](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#eviction-signals) with either percentage values or resource values. - -```yaml -kubelet: - evictionHard: - memory.available: 500Mi - nodefs.available: 10% - nodefs.inodesFree: 10% - imagefs.available: 5% - imagefs.inodesFree: 5% - pid.available: 7% - evictionSoft: - memory.available: 1Gi - nodefs.available: 15% - nodefs.inodesFree: 15% - imagefs.available: 10% - imagefs.inodesFree: 10% - pid.available: 10% -``` - -#### Supported Eviction Signals - -| Eviction Signal | Description | -|--------------------|---------------------------------------------------------------------------------| -| memory.available | memory.available := node.status.capacity[memory] - node.stats.memory.workingSet | -| nodefs.available | nodefs.available := node.stats.fs.available | -| nodefs.inodesFree | nodefs.inodesFree := node.stats.fs.inodesFree | -| imagefs.available | imagefs.available := node.stats.runtime.imagefs.available | -| imagefs.inodesFree | imagefs.inodesFree := node.stats.runtime.imagefs.inodesFree | -| pid.available | pid.available := node.stats.rlimit.maxpid - node.stats.rlimit.curproc | - -For more information on eviction thresholds, view the [Node-pressure Eviction](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction) section of the official Kubernetes docs. - -#### Soft Eviction Grace Periods - -Soft eviction pairs an eviction threshold with a specified grace period. With soft eviction thresholds, the kubelet will only begin evicting pods when the node exceeds its soft eviction threshold over the entire duration of its grace period. For example, if you specify `evictionSoft[memory.available]` of `500Mi` and a `evictionSoftGracePeriod[memory.available]` of `1m30`, the node must have less than `500Mi` of available memory over a minute and a half in order for the kubelet to begin evicting pods. - -Optionally, you can specify an `evictionMaxPodGracePeriod` which defines the administrator-specified maximum pod termination grace period to use during soft eviction. If a namespace-owner had specified a pod `terminationGracePeriodInSeconds` on pods in their namespace, the minimum of `evictionPodGracePeriod` and `terminationGracePeriodInSeconds` would be used. - -```yaml -kubelet: - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - imagefs.available: 1m30s - imagefs.inodesFree: 2m - pid.available: 2m - evictionMaxPodGracePeriod: 60 -``` - -### Pod Density - -By default, the number of pods on a node is limited by both the number of networking interfaces (ENIs) that may be attached to an instance type and the number of IP addresses that can be assigned to each ENI. See [IP addresses per network interface per instance type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI) for a more detailed information on these instance types' limits. - -{{% alert title="Note" color="primary" %}} -By default, the VPC CNI allocates IPs for a node and pods from the same subnet. With [VPC CNI Custom Networking](https://aws.github.io/aws-eks-best-practices/networking/custom-networking), the pods will receive IP addresses from another subnet dedicated to pod IPs. This approach makes it easier to manage IP addresses and allows for separate Network Access Control Lists (NACLs) applied to your pods. VPC CNI Custom Networking reduces the pod density of a node since one of the ENI attachments will be used for the node and cannot share the allocated IPs on the interface to pods. Karpenter supports VPC CNI Custom Networking and similar CNI setups where the primary node interface is separated from the pods interfaces through a global [setting](./settings.md#configmap) within the karpenter-global-settings configmap: `aws.reservedENIs`. In the common case, `aws.reservedENIs` should be set to `"1"` if using Custom Networking. -{{% /alert %}} - -{{% alert title="Windows Support Notice" color="warning" %}} -It's currently not possible to specify custom networking with Windows nodes. -{{% /alert %}} - -#### Max Pods - -For small instances that require an increased pod density or large instances that require a reduced pod density, you can override this default value with `.spec.template.spec.kubelet.maxPods`. This value will be used during Karpenter pod scheduling and passed through to `--max-pods` on kubelet startup. - -{{% alert title="Note" color="primary" %}} -When using small instance types, it may be necessary to enable [prefix assignment mode](https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/) in the AWS VPC CNI plugin to support a higher pod density per node. Prefix assignment mode was introduced in AWS VPC CNI v1.9 and allows ENIs to manage a broader set of IP addresses. Much higher pod densities are supported as a result. -{{% /alert %}} - -{{% alert title="Windows Support Notice" color="warning" %}} -Presently, Windows worker nodes do not support using more than one ENI. -As a consequence, the number of IP addresses, and subsequently, the number of pods that a Windows worker node can support is limited by the number of IPv4 addresses available on the primary ENI. -Currently, Karpenter will only consider individual secondary IP addresses when calculating the pod density limit. -{{% /alert %}} - -#### Pods Per Core - -An alternative way to dynamically set the maximum density of pods on a node is to use the `.spec.template.spec.kubelet.podsPerCore` value. Karpenter will calculate the pod density during scheduling by multiplying this value by the number of logical cores (vCPUs) on an instance type. This value will also be passed through to the `--pods-per-core` value on kubelet startup to configure the number of allocatable pods the kubelet can assign to the node instance. - -The value generated from `podsPerCore` cannot exceed `maxPods`, meaning, if both are set, the minimum of the `podsPerCore` dynamic pod density and the static `maxPods` value will be used for scheduling. - -{{% alert title="Note" color="primary" %}} -`maxPods` may not be set in the `kubelet` of a NodePool, but may still be restricted by the `ENI_LIMITED_POD_DENSITY` value. You may want to ensure that the `podsPerCore` value that will be used for instance families associated with the NodePool will not cause unexpected behavior by exceeding the `maxPods` value. -{{% /alert %}} - -{{% alert title="Pods Per Core on Bottlerocket" color="warning" %}} -Bottlerocket AMIFamily currently does not support `podsPerCore` configuration. If a NodePool contains a `provider` or `providerRef` to a node template that will launch a Bottlerocket instance, the `podsPerCore` value will be ignored for scheduling and for configuring the kubelet. -{{% /alert %}} ## spec.disruption -You can configure Karpenter to disrupt Nodes through your NodePool in multiple ways. You can use `spec.disruption.consolidationPolicy`, `spec.disruption.consolidateAfter` or `spec.disruption.expireAfter`. Read [Disruption]({{}}) for more. +You can configure Karpenter to disrupt Nodes through your NodePool in multiple ways. You can use `spec.disruption.consolidationPolicy`, `spec.disruption.consolidateAfter`, or `spec.template.spec.expireAfter`. +You can also rate limit Karpenter's disruption through the NodePool's `spec.disruption.budgets`. +Read [Disruption]({{}}) for more. ## spec.limits @@ -486,7 +370,7 @@ The NodePool spec includes a limits section (`spec.limits`), which constrains th Karpenter supports limits of any resource type reported by your cloudprovider. It limits instance types when scheduling to those that will not exceed the specified limits. If a limit has been exceeded, nodes provisioning is prevented until some nodes have been terminated. ```yaml -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -524,6 +408,18 @@ Karpenter allows you to describe NodePool preferences through a `weight` mechani For more information on weighting NodePools, see the [Weighted NodePools section]({{}}) in the scheduling docs. +## status.conditions +[Conditions](https://github.com/kubernetes/apimachinery/blob/f14778da5523847e4c07346e3161a4b4f6c9186e/pkg/apis/meta/v1/types.go#L1523) objects add observability features to Karpenter. +* The `status.conditions.type` object reflects node status, such as `Initialized` or `Available`. +* The status of the condition, `status.conditions.status`, indicates if the condition is `True` or `False`. +* The `status.conditions.observedGeneration` indicates if the instance is out of date with the current state of `.metadata.generation`. +* The `status.conditions.lastTransitionTime` object contains a programatic identifier that indicates the time of the condition's previous transition. +* The `status.conditions.reason` object indicates the reason for the condition's previous transition. +* The `status.conditions.message` object provides human-readable details about the condition's previous transition. + +## status.resources +Objects under `status.resources` provide information about the status of resources such as `cpu`, `memory`, and `ephemeral-storage`. + ## Examples ### Isolating Expensive Hardware @@ -532,13 +428,13 @@ A NodePool can be set up to only provision nodes on particular processor types. The following example sets a taint that only allows pods with tolerations for Nvidia GPUs to be scheduled: ```yaml -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: gpu spec: disruption: - consolidationPolicy: WhenUnderutilized + consolidationPolicy: WhenEmptyOrUnderutilized template: spec: requirements: @@ -556,14 +452,16 @@ In order for a pod to run on a node defined in this NodePool, it must tolerate ` Per the Cilium [docs](https://docs.cilium.io/en/stable/installation/taints/#taint-effects), it's recommended to place a taint of `node.cilium.io/agent-not-ready=true:NoExecute` on nodes to allow Cilium to configure networking prior to other pods starting. This can be accomplished via the use of Karpenter `startupTaints`. These taints are placed on the node, but pods aren't required to tolerate these taints to be considered for provisioning. +Failure to provide accurate `startupTaints` can result in Karpenter continually provisioning new nodes. When the new node joins and the startup taint that Karpenter is unaware of is added, Karpenter now considers the pending pod to be unschedulable to this node. Karpenter will attempt to provision yet another new node to schedule the pending pod. + ```yaml -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: cilium-startup spec: disruption: - consolidationPolicy: WhenUnderutilized + consolidationPolicy: WhenEmptyOrUnderutilized template: spec: startupTaints: diff --git a/website/content/en/preview/concepts/scheduling.md b/website/content/en/preview/concepts/scheduling.md index 76c09cbc5124..2ef5b2c62897 100755 --- a/website/content/en/preview/concepts/scheduling.md +++ b/website/content/en/preview/concepts/scheduling.md @@ -1,7 +1,7 @@ --- title: "Scheduling" linkTitle: "Scheduling" -weight: 3 +weight: 40 description: > Learn about scheduling workloads with Karpenter --- @@ -104,8 +104,6 @@ Refer to general [Kubernetes GPU](https://kubernetes.io/docs/tasks/manage-gpus/s You must enable Pod ENI support in the AWS VPC CNI Plugin before enabling Pod ENI support in Karpenter. Please refer to the [Security Groups for Pods documentation](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) for instructions. {{% /alert %}} -Now that Pod ENI support is enabled in the AWS VPC CNI Plugin, you can enable Pod ENI support in Karpenter by setting the `settings.aws.enablePodENI` Helm chart value to `true`. - Here is an example of a pod-eni resource defined in a deployment manifest: ``` spec: @@ -154,6 +152,7 @@ Take care to ensure the label domains are correct. A well known label like `karp | karpenter.k8s.aws/instance-cpu | 32 | [AWS Specific] Number of CPUs on the instance | | karpenter.k8s.aws/instance-cpu-manufacturer | aws | [AWS Specific] Name of the CPU manufacturer | | karpenter.k8s.aws/instance-memory | 131072 | [AWS Specific] Number of mebibytes of memory on the instance | +| karpenter.k8s.aws/instance-ebs-bandwidth | 9500 | [AWS Specific] Number of [maximum megabits](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-optimized.html#ebs-optimization-performance) of EBS available on the instance | | karpenter.k8s.aws/instance-network-bandwidth | 131072 | [AWS Specific] Number of [baseline megabits](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html) available on the instance | | karpenter.k8s.aws/instance-pods | 110 | [AWS Specific] Number of pods the instance supports | | karpenter.k8s.aws/instance-gpu-name | t4 | [AWS Specific] Name of the GPU on the instance, if available | @@ -176,6 +175,10 @@ requirements: operator: Exists ``` +{{% alert title="Note" color="primary" %}} +There is currently a limit of 100 on the total number of requirements on both the NodePool and the NodeClaim. It's important to note that `spec.template.metadata.labels` are also propagated as requirements on the NodeClaim when it's created, meaning that you can't have more than 100 requirements and labels combined set on your NodePool. +{{% /alert %}} + #### Node selectors Here is an example of a `nodeSelector` for selecting nodes: @@ -193,6 +196,16 @@ Then the pod can declare that custom label. See [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) in the Kubernetes documentation for details. +## Preferences + +Karpenter is aware of preferences (node affinity, pod affinity, pod anti-affinity, and pod topology) and treats them as requirements in most circumstances. Karpenter uses these preferences when determining if a pod can schedule on a node (absent topology requirements), or when determining if a pod can be shifted to a new node. + +Karpenter starts by treating preferred affinities as required affinities when constructing requirements for a pod. When these requirements cannot be met, the pod's preferences are relaxed one-at-a-time by ascending weight (lowest weight is relaxed first), and the remaining requirements are tried again. + +{{% alert title="Warning" color="warning" %}} +Karpenter does not interpret preferred affinities as required when constructing topology requirements for scheduling to a node. If these preferences are necessary, required affinities should be used [as documented in Node Affinity](#node-affinity). +{{% /alert %}} + ### Node affinity Examples below illustrate how to use Node affinity to include (`In`) and exclude (`NotIn`) objects. @@ -202,6 +215,10 @@ When setting rules, the following Node affinity types define how hard or soft ea * **requiredDuringSchedulingIgnoredDuringExecution**: This is a hard rule that must be met. * **preferredDuringSchedulingIgnoredDuringExecution**: This is a preference, but the pod can run on a node where it is not guaranteed. +{{% alert title="Note" color="primary" %}} +Preferred affinities on pods can result in more nodes being created than expected because Karpenter will prefer to create new nodes to satisfy preferences, [see the preferences documentation](#preferences) for details. +{{% /alert %}} + The `IgnoredDuringExecution` part of each tells the pod to keep running, even if conditions change on the node so the rules no longer matched. You can think of these concepts as `required` and `preferred`, since Kubernetes never implemented other variants of these rules. @@ -262,13 +279,13 @@ If they all fail, Karpenter will fail to provision the pod. Karpenter will backoff and retry over time. So if capacity becomes available, it will schedule the pod without user intervention. -## Taints and tolerations +### Taints and tolerations Taints are the opposite of affinity. Setting a taint on a node tells the scheduler to not run a pod on it unless the pod has explicitly said it can tolerate that taint. This example shows a NodePool that was set up with a taint for only running pods that require a GPU, such as the following: ```yaml -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: gpu @@ -282,7 +299,7 @@ spec: - p3 taints: - key: nvidia.com/gpu - value: true + value: "true" effect: "NoSchedule" ``` @@ -309,9 +326,14 @@ spec: ``` See [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) in the Kubernetes documentation for details. -## Topology Spread +### Topology Spread By using the Kubernetes `topologySpreadConstraints` you can ask the NodePool to have pods push away from each other to limit the blast radius of an outage. Think of it as the Kubernetes evolution for pod affinity: it lets you relate pods with respect to nodes while still allowing spread. + +{{% alert title="Note" color="primary" %}} +Preferred topology spread (`ScheduleAnyway`) can result in more nodes being created than expected because Karpenter will prefer to create new nodes to satisfy spread constraints, [see the preferences documentation](#preferences) for details. +{{% /alert %}} + For example: ```yaml @@ -356,9 +378,15 @@ See [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/worklo NodePools do not attempt to balance or rebalance the availability zones for their nodes. Availability zone balancing may be achieved by defining zonal Topology Spread Constraints for Pods that require multi-zone durability, and NodePools will respect these constraints while optimizing for compute costs. {{% /alert %}} -## Pod affinity/anti-affinity +### Pod affinity/anti-affinity + +By using the `podAffinity` and `podAntiAffinity` configuration on a pod spec, you can inform the Karpenter scheduler of your desire for pods to schedule together or apart with respect to different topology domains. + +{{% alert title="Note" color="primary" %}} +Preferred affinities on pods can result in more nodes being created than expected because Karpenter will prefer to create new nodes to satisfy preferences, [see the preferences documentation](#preferences) for details. +{{% /alert %}} -By using the `podAffinity` and `podAntiAffinity` configuration on a pod spec, you can inform the Karpenter scheduler of your desire for pods to schedule together or apart with respect to different topology domains. For example: +For example: ```yaml spec: @@ -386,7 +414,7 @@ The anti-affinity rule would cause it to avoid running on any node with a pod la See [Inter-pod affinity and anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity) in the Kubernetes documentation for details. -## Persistent Volume Topology +### Persistent Volume Topology Karpenter automatically detects storage scheduling requirements and includes them in node launch decisions. @@ -454,7 +482,7 @@ If you have purchased a [Savings Plan](https://aws.amazon.com/savingsplans/) or To enable this, you will need to tell the Karpenter controllers which instance types to prioritize and what is the maximum amount of capacity that should be provisioned using those instance types. We can set the `.spec.limits` field on the NodePool to limit the capacity that can be launched by this NodePool. Combined with the `.spec.weight` value, we can tell Karpenter to pull from instance types in the reserved NodePool before defaulting to generic instance types. ```yaml -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: reserved-instance @@ -469,7 +497,7 @@ spec: operator: In values: ["c4.large"] --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -492,7 +520,7 @@ Pods that do not specify node selectors or affinities can potentially be assigne By assigning a higher `.spec.weight` value and restricting a NodePool to a specific capacity type or architecture, we can set default configuration for the nodes launched by pods that don't have node configuration restrictions. ```yaml -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -508,7 +536,7 @@ spec: operator: In values: ["amd64"] --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: arm64-specific @@ -626,19 +654,73 @@ If using Gt/Lt operators, make sure to use values under the actual label values The `Exists` operator can be used on a NodePool to provide workload segregation across nodes. ```yaml -... -requirements: -- key: company.com/team - operator: Exists +apiVersion: karpenter.sh/v1 +kind: NodePool +spec: + template: + spec: + requirements: + - key: company.com/team + operator: Exists ... ``` -With the requirement on the NodePool, workloads can optionally specify a custom value as a required node affinity or node selector. Karpenter will then label the nodes it launches for these pods which prevents `kube-scheduler` from scheduling conflicting pods to those nodes. This provides a way to more dynamically isolate workloads without requiring a unique NodePool for each workload subset. +With this requirement on the NodePool, workloads can specify the same key (e.g. `company.com/team`) with custom values (e.g. `team-a`, `team-b`, etc.) as a required `nodeAffinity` or `nodeSelector`. Karpenter will then apply the key/value pair to nodes it launches dynamically based on the pod's node requirements. + +If each set of pods that can schedule with this NodePool specifies this key in its `nodeAffinity` or `nodeSelector`, you can isolate pods onto different nodes based on their values. This provides a way to more dynamically isolate workloads without requiring a unique NodePool for each workload subset. + +For example, providing the following `nodeSelectors` would isolate the pods for each of these deployments on different nodes. + +#### Team A Deployment ```yaml -nodeSelector: - company.com/team: team-a +apiVersion: v1 +kind: Deployment +metadata: + name: team-a-deployment +spec: + replicas: 5 + template: + spec: + nodeSelector: + company.com/team: team-a ``` + +#### Team A Node + +```yaml +apiVersion: v1 +kind: Node +metadata: + labels: + company.com/team: team-a +``` + +#### Team B Deployment + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: team-b-deployment +spec: + replicas: 5 + template: + spec: + nodeSelector: + company.com/team: team-b +``` + +#### Team B Node + +```yaml +apiVersion: v1 +kind: Node +metadata: + labels: + company.com/team: team-b +``` + {{% alert title="Note" color="primary" %}} If a workload matches the NodePool but doesn't specify a label, Karpenter will generate a random label for the node. {{% /alert %}} @@ -656,7 +738,7 @@ This is not identical to a topology spread with a specified ratio. We are const #### NodePools ```yaml -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: spot @@ -675,7 +757,7 @@ spec: - "4" - "5" --- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: on-demand diff --git a/website/content/en/preview/contributing/development-guide.md b/website/content/en/preview/contributing/development-guide.md index 93796441ea98..02632af9222e 100644 --- a/website/content/en/preview/contributing/development-guide.md +++ b/website/content/en/preview/contributing/development-guide.md @@ -84,13 +84,13 @@ By default, `make apply` will set the log level to debug. You can change the log OSX: ```bash -open http://localhost:8000/metrics && kubectl port-forward service/karpenter -n karpenter 8000 +open http://localhost:8080/metrics && kubectl port-forward service/karpenter -n kube-system 8080 ``` Linux: ```bash -gio open http://localhost:8000/metrics && kubectl port-forward service/karpenter -n karpenter 8000 +gio open http://localhost:8080/metrics && kubectl port-forward service/karpenter -n karpenter 8080 ``` ### Tailing Logs @@ -143,8 +143,8 @@ go install github.com/google/pprof@latest ### Get a profile ``` # Connect to the metrics endpoint -kubectl port-forward service/karpenter -n karpenter 8000 -open http://localhost:8000/debug/pprof/ +kubectl port-forward service/karpenter -n karpenter 8080 +open http://localhost:8080/debug/pprof/ # Visualize the memory -go tool pprof -http 0.0.0.0:9000 localhost:8000/debug/pprof/heap +go tool pprof -http 0.0.0.0:9000 localhost:8080/debug/pprof/heap ``` diff --git a/website/content/en/preview/contributing/documentation-updates.md b/website/content/en/preview/contributing/documentation-updates.md new file mode 100644 index 000000000000..daa6516e30b1 --- /dev/null +++ b/website/content/en/preview/contributing/documentation-updates.md @@ -0,0 +1,11 @@ +--- +title: "Documentation Updates" +linkTitle: "Documentation Updates" +weight: 50 +description: > + Infomration helpful for contributing simple documentation updates. +--- + +- Documentation for https://karpenter.sh/docs/ is built under website/content/en/preview/. +- Documentation updates should be made to the "preview" directory. Your changes will be promoted to website/content/en/docs/ by an automated process after the change has been merged. +- Previews for your changes are built and available a few minutes after you push. Look for the "netlify Deploy Preview" link in a comment in your PR. diff --git a/website/content/en/preview/faq.md b/website/content/en/preview/faq.md index 4549a26bef94..48973e94f401 100644 --- a/website/content/en/preview/faq.md +++ b/website/content/en/preview/faq.md @@ -7,6 +7,9 @@ description: > --- ## General +### Is Karpenter safe for production use? +Karpenter v1 is the first stable Karpenter API. Any future incompatible API changes will require a v2 version. + ### How does a NodePool decide to manage a particular node? See [Configuring NodePools]({{< ref "./concepts/#configuring-nodepools" >}}) for information on how Karpenter configures and manages nodes. @@ -17,7 +20,7 @@ AWS is the first cloud provider supported by Karpenter, although it is designed Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree{{< githubRelRef >}}pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. ### What operating system nodes does Karpenter deploy? -Karpenter uses the OS defined by the [AMI Family in your EC2NodeClass]({{< ref "./concepts/nodeclasses#specamifamily" >}}). +Karpenter uses the OS defined by the [AMI Family in your EC2NodeClass]({{< ref "./concepts/nodeclasses#specamifamily" >}}). ### Can I provide my own custom operating system images? Karpenter has multiple mechanisms for configuring the [operating system]({{< ref "./concepts/nodeclasses/#specamiselectorterms" >}}) for your nodes. @@ -119,7 +122,7 @@ Karpenter has a concept of an “offering” for each instance type, which is a Yes! Karpenter dynamically discovers if you are running in an IPv6 cluster by checking the kube-dns service's cluster-ip. When using an AMI Family such as `AL2`, Karpenter will automatically configure the EKS Bootstrap script for IPv6. Some EC2 instance types do not support IPv6 and the Amazon VPC CNI only supports instance types that run on the Nitro hypervisor. It's best to add a requirement to your NodePool to only allow Nitro instance types: ``` -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool ... spec: @@ -138,6 +141,25 @@ For more documentation on enabling IPv6 with the Amazon VPC CNI, see the [docs]( Windows nodes do not support IPv6. {{% /alert %}} +### Why do I see extra nodes get launched to schedule pending pods that remain empty and are later removed? + +You might have a daemonset, userData configuration, or some other workload that applies a taint after a node is provisioned. After the taint is applied, Karpenter will detect that the pod cannot be scheduled to this new node due to the added taint. As a result, Karpenter will provision yet another node. Typically, the original node has the taint removed and the pod schedules to it, leaving the extra new node unused and reaped by emptiness/consolidation. If the taint is not removed quickly enough, Karpenter may remove the original node before the pod can be scheduled via emptiness consolidation. This could result in an infinite loop of nodes being provisioned and consolidated without the pending pod ever scheduling. + +The solution is to configure [startupTaints]({{}}) to make Karpenter aware of any temporary taints that are needed to ensure that pods do not schedule on nodes that are not yet ready to receive them. + +Here's an example for Cilium's startup taint. +``` +apiVersion: karpenter.sh/v1 +kind: NodePool +... +spec: + template: + spec: + startupTaints: + - key: node.cilium.io/agent-not-ready + effect: NoSchedule +``` + ## Scheduling ### When using preferred scheduling constraints, Karpenter launches the correct number of nodes at first. Why do they then sometimes get consolidated immediately? @@ -211,15 +233,15 @@ For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgra ### How do I upgrade an EKS Cluster with Karpenter? -When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift is enabled by default starting `0.33.0`. - {{% alert title="Note" color="primary" %}} -Karpenter's default [EC2NodeClass `amiFamily` configuration]({{}}) uses the latest EKS Optimized AL2 AMI for the same major and minor version as the EKS cluster's control plane, meaning that an upgrade of the control plane will cause Karpenter to auto-discover the new AMIs for that version. +Karpenter recommends that you always validate AMIs in your lower environments before using them in production environments. Read [Managing AMIs]({{}}) to understand best practices about upgrading your AMIs. -If using a custom AMI, you will need to trigger the rollout of this new worker node image through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. +If using a custom AMI, you will need to trigger the rollout of new worker node images through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. {{% /alert %}} -Start by [upgrading the EKS Cluster control plane](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). After the EKS Cluster upgrade completes, Karpenter's Drift feature will detect that the Karpenter-provisioned nodes are using EKS Optimized AMIs for the previous cluster version, and [automatically cordon, drain, and replace those nodes]({{}}). To support pods moving to new nodes, follow Kubernetes best practices by setting appropriate pod [Resource Quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/), and using [Pod Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (PDB). Karpenter's Drift feature will spin up replacement nodes based on the pod resource requests, and will respect the PDBs when deprovisioning nodes. +Karpenter's default behavior will upgrade your nodes when you've upgraded your Amazon EKS Cluster. Karpenter will [drift]({{}}) nodes to stay in-sync with the EKS control plane version. Drift is enabled by default starting in `v0.33`. This means that as soon as your cluster is upgraded, Karpenter will auto-discover the new AMIs for that version. + +Start by [upgrading the EKS Cluster control plane](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). After the EKS Cluster upgrade completes, Karpenter will Drift and disrupt the Karpenter-provisioned nodes using EKS Optimized AMIs for the previous cluster version by first spinning up replacement nodes. Karpenter respects [Pod Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (PDB), and automatically [replaces, cordons, and drains those nodes]({{}}). To best support pods moving to new nodes, follow Kubernetes best practices by setting appropriate pod [Resource Quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/) and using PDBs. ## Interruption Handling @@ -231,7 +253,7 @@ Karpenter's native interruption handling offers two main benefits over the stand 1. You don't have to manage and maintain a separate component to exclusively handle interruption events. 2. Karpenter's native interruption handling coordinates with other deprovisioning so that consolidation, expiration, etc. can be aware of interruption events and vice-versa. -### Why am I receiving QueueNotFound errors when I set `--interruption-queue-name`? +### Why am I receiving QueueNotFound errors when I set `--interruption-queue`? Karpenter requires a queue to exist that receives event messages from EC2 and health services in order to handle interruption messages properly for nodes. Details on the types of events that Karpenter handles can be found in the [Interruption Handling Docs]({{< ref "./concepts/disruption/#interruption" >}}). diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md index 7688f0a8155e..e2b943108a1b 100644 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md @@ -11,7 +11,9 @@ Karpenter automatically provisions new nodes in response to unschedulable pods. This guide shows how to get started with Karpenter by creating a Kubernetes cluster and installing Karpenter. To use Karpenter, you must be running a supported Kubernetes cluster on a supported cloud provider. -Currently, only EKS on AWS is supported. +Currently, the following Cloud Providers are supported: +- [AWS](https://github.com/aws/karpenter-provider-aws) +- [Azure](https://github.com/Azure/karpenter-provider-azure) ## Create a cluster and add Karpenter @@ -32,7 +34,7 @@ Install these tools before proceeding: 1. [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html) 2. `kubectl` - [the Kubernetes CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) -3. `eksctl` (>= v0.169.0) - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) +3. `eksctl` (>= v0.180.0) - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) 4. `helm` - [the package manager for Kubernetes](https://helm.sh/docs/intro/install/) [Configure the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) @@ -87,6 +89,9 @@ The following cluster configuration will: {{% /tab %}} {{< /tabpane >}} +Unless your AWS account has already onboarded to EC2 Spot, you will need to create the service linked role to +avoid the [`ServiceLinkedRoleCreationNotPermitted` error]({{}}). + {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh" language="bash"%}} {{% alert title="Windows Support Notice" color="warning" %}} @@ -148,7 +153,7 @@ A single Karpenter NodePool is capable of handling many different pod shapes. Ka Create a default NodePool using the command below. This NodePool uses `securityGroupSelectorTerms` and `subnetSelectorTerms` to discover resources used to launch nodes. We applied the tag `karpenter.sh/discovery` in the `eksctl` command above. Depending on how these resources are shared between clusters, you may need to use different tagging schemes. -The `consolidationPolicy` set to `WhenUnderutilized` in the `disruption` block configures Karpenter to reduce cluster cost by removing and replacing nodes. As a result, consolidation will terminate any empty nodes on the cluster. This behavior can be disabled by setting `consolidateAfter` to `Never`, telling Karpenter that it should never consolidate nodes. Review the [NodePool API docs]({{}}) for more information. +The `consolidationPolicy` set to `WhenEmptyOrUnderutilized` in the `disruption` block configures Karpenter to reduce cluster cost by removing and replacing nodes. As a result, consolidation will terminate any empty nodes on the cluster. This behavior can be disabled by setting `consolidateAfter` to `Never`, telling Karpenter that it should never consolidate nodes. Review the [NodePool API docs]({{}}) for more information. Note: This NodePool will create capacity as long as the sum of all created capacity is less than the specified limit. diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml b/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml index f80755267455..567808be5830 100644 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml @@ -83,7 +83,8 @@ Resources: ], "Condition": { "StringEquals": { - "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned" + "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:RequestTag/eks:eks-cluster-name": "${ClusterName}" }, "StringLike": { "aws:RequestTag/karpenter.sh/nodepool": "*" @@ -105,6 +106,7 @@ Resources: "Condition": { "StringEquals": { "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:RequestTag/eks:eks-cluster-name": "${ClusterName}", "ec2:CreateAction": [ "RunInstances", "CreateFleet", @@ -128,8 +130,12 @@ Resources: "StringLike": { "aws:ResourceTag/karpenter.sh/nodepool": "*" }, + "StringEqualsIfExists": { + "aws:RequestTag/eks:eks-cluster-name": "${ClusterName}" + }, "ForAllValues:StringEquals": { "aws:TagKeys": [ + "eks:eks-cluster-name", "karpenter.sh/nodeclaim", "Name" ] @@ -213,13 +219,14 @@ Resources: { "Sid": "AllowScopedInstanceProfileCreationActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:CreateInstanceProfile" ], "Condition": { "StringEquals": { "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:RequestTag/eks:eks-cluster-name": "${ClusterName}", "aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}" }, "StringLike": { @@ -230,7 +237,7 @@ Resources: { "Sid": "AllowScopedInstanceProfileTagActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:TagInstanceProfile" ], @@ -239,6 +246,7 @@ Resources: "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned", "aws:ResourceTag/topology.kubernetes.io/region": "${AWS::Region}", "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:RequestTag/eks:eks-cluster-name": "${ClusterName}", "aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}" }, "StringLike": { @@ -250,7 +258,7 @@ Resources: { "Sid": "AllowScopedInstanceProfileActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:AddRoleToInstanceProfile", "iam:RemoveRoleFromInstanceProfile", @@ -269,7 +277,7 @@ Resources: { "Sid": "AllowInstanceProfileReadActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": "iam:GetInstanceProfile" }, { @@ -301,6 +309,14 @@ Resources: - sqs.amazonaws.com Action: sqs:SendMessage Resource: !GetAtt KarpenterInterruptionQueue.Arn + - Sid: DenyHTTP + Effect: Deny + Action: sqs:* + Resource: !GetAtt KarpenterInterruptionQueue.Arn + Condition: + Bool: + aws:SecureTransport: false + Principal: "*" ScheduledChangeRule: Type: 'AWS::Events::Rule' Properties: @@ -344,4 +360,4 @@ Resources: - EC2 Instance State-change Notification Targets: - Id: KarpenterInterruptionQueueTarget - Arn: !GetAtt KarpenterInterruptionQueue.Arn \ No newline at end of file + Arn: !GetAtt KarpenterInterruptionQueue.Arn diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json b/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json index d474d01f4e16..e85e582de299 100644 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json @@ -1,1440 +1,1602 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 2, - "id": 6, - "links": [], - "liveNow": true, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(action, cluster) (karpenter_deprovisioning_actions_performed)", - "format": "time_series", - "instant": false, - "legendFormat": "{{cluster}}: {{action}}", - "range": true, - "refId": "A" - } - ], - "title": "Deprovisioning Actions Performed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "sum by(cluster) (karpenter_nodes_created)", - "format": "time_series", - "legendFormat": "{{cluster}}", - "range": true, - "refId": "A" - } - ], - "title": "Nodes Created", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 15, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "sum by(cluster) (karpenter_nodes_terminated)", - "format": "time_series", - "legendFormat": "{{cluster}}", - "range": true, - "refId": "A" - } - ], - "title": "Nodes Terminated", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by(phase)(karpenter_pods_state)", - "legendFormat": "{{label_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Pod Phase", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", - "legendFormat": "{{label_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Pod Distribution: $distribution_filter", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-RdYlGr" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*Utilization$" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 1 - }, - { - "id": "unit", - "value": "percentunit" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Provisioned" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 18, - "x": 0, - "y": 29 - }, - "id": 10, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "9.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "karpenter_nodepool_usage{resource_type=\"cpu\"} / karpenter_nodepool_limit{resource_type=\"cpu\"}", - "format": "table", - "instant": true, - "legendFormat": "CPU Limit Utilization", - "range": false, - "refId": "CPU Limit Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Node Count" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "karpenter_nodepool_usage{resource_type=\"memory\"} / karpenter_nodepool_limit{resource_type=\"memory\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Memory Limit Utilization", - "range": false, - "refId": "Memory Limit Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "CPU Capacity" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Memory Capacity" - } - ], - "title": "Nodepool Summary", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "nodepool" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "__name__": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "job": true, - "job 1": true, - "job 2": true, - "resource_type": true, - "resource_type 1": true, - "resource_type 2": true - }, - "indexByName": { - "Time 1": 6, - "Time 2": 7, - "Time 3": 11, - "Time 4": 15, - "Time 5": 16, - "Value #CPU Capacity": 2, - "Value #CPU Limit Utilization": 3, - "Value #Memory Capacity": 4, - "Value #Memory Limit Utilization": 5, - "Value #Node Count": 1, - "instance 1": 8, - "instance 2": 12, - "job 1": 9, - "job 2": 13, - "nodepool": 0, - "resource_type 1": 10, - "resource_type 2": 14 - }, - "renameByName": { - "Time 1": "", - "Value": "CPU Utilization", - "Value #CPU Capacity": "CPU Provisioned", - "Value #CPU Limit Utilization": "CPU Limit Utilization", - "Value #CPU Utilization": "CPU Limit Utilization", - "Value #Memory Capacity": "Memory Provisioned", - "Value #Memory Limit Utilization": "Memory Limit Utilization", - "Value #Memory Utilization": "Memory Utilization", - "Value #Node Count": "Node Count", - "instance": "", - "instance 1": "", - "job": "", - "nodepool": "Nodepool" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 18, - "y": 29 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", - "legendFormat": "Percentage", - "range": true, - "refId": "A" - } - ], - "title": "Spot Node Percentage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-RdYlGr" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "node_name" - }, - "properties": [ - { - "id": "custom.width", - "value": 333 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Utilization" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "thresholds", - "value": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 75 - } - ] - } - }, - { - "id": "max", - "value": 1 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "unit", - "value": "s" - }, - { - "id": "decimals", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 4, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Uptime" - } - ] - }, - "pluginVersion": "9.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "CPU Utilization", - "range": false, - "refId": "CPU Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Memory Utilization", - "range": false, - "refId": "Memory Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Memory Utilization", - "range": false, - "refId": "Pod Count" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\n sum by (node)(node_time_seconds) - sum by (node)(node_boot_time_seconds),\n \"node_name\", \"$1\", \"node\", \"(.+)\"\n)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Uptime", - "range": false, - "refId": "Uptime" - } - ], - "title": "Node Summary", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "node_name" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Value": false, - "Value #Pod Count": false, - "__name__": true, - "arch": true, - "arch 1": true, - "arch 2": true, - "arch 3": true, - "capacity_type 2": true, - "capacity_type 3": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "instance 3": true, - "instance_category 1": true, - "instance_category 2": true, - "instance_category 3": true, - "instance_cpu": true, - "instance_cpu 1": true, - "instance_cpu 2": true, - "instance_cpu 3": true, - "instance_family": true, - "instance_family 1": true, - "instance_family 2": true, - "instance_family 3": true, - "instance_generation 1": true, - "instance_generation 2": true, - "instance_generation 3": true, - "instance_gpu_count": true, - "instance_gpu_count 1": true, - "instance_gpu_count 2": true, - "instance_gpu_count 3": true, - "instance_gpu_manufacturer": true, - "instance_gpu_manufacturer 1": true, - "instance_gpu_manufacturer 2": true, - "instance_gpu_manufacturer 3": true, - "instance_gpu_memory": true, - "instance_gpu_memory 1": true, - "instance_gpu_memory 2": true, - "instance_gpu_memory 3": true, - "instance_gpu_name": true, - "instance_gpu_name 1": true, - "instance_gpu_name 2": true, - "instance_gpu_name 3": true, - "instance_hypervisor": true, - "instance_hypervisor 1": true, - "instance_hypervisor 2": true, - "instance_hypervisor 3": true, - "instance_local_nvme 1": true, - "instance_local_nvme 2": true, - "instance_local_nvme 3": true, - "instance_memory": true, - "instance_memory 1": true, - "instance_memory 2": true, - "instance_memory 3": true, - "instance_pods": true, - "instance_pods 1": true, - "instance_pods 2": true, - "instance_pods 3": true, - "instance_size": true, - "instance_size 1": true, - "instance_size 2": true, - "instance_size 3": true, - "instance_type 1": false, - "instance_type 2": true, - "instance_type 3": true, - "job": true, - "job 1": true, - "job 2": true, - "job 3": true, - "node": true, - "os": true, - "os 1": true, - "os 2": true, - "os 3": true, - "nodepool 1": false, - "nodepool 2": true, - "nodepool 3": true, - "resource_type": true, - "resource_type 1": true, - "resource_type 2": true, - "resource_type 3": true, - "zone 1": false, - "zone 2": true, - "zone 3": true - }, - "indexByName": { - "Time 1": 1, - "Time 2": 25, - "Time 3": 45, - "Time 4": 65, - "Value #CPU Utilization": 10, - "Value #Memory Utilization": 11, - "Value #Pod Count": 9, - "Value #Uptime": 8, - "arch 1": 5, - "arch 2": 26, - "arch 3": 46, - "capacity_type 1": 6, - "capacity_type 2": 27, - "capacity_type 3": 47, - "instance 1": 4, - "instance 2": 28, - "instance 3": 48, - "instance_cpu 1": 12, - "instance_cpu 2": 29, - "instance_cpu 3": 49, - "instance_family 1": 13, - "instance_family 2": 30, - "instance_family 3": 50, - "instance_gpu_count 1": 14, - "instance_gpu_count 2": 31, - "instance_gpu_count 3": 51, - "instance_gpu_manufacturer 1": 15, - "instance_gpu_manufacturer 2": 32, - "instance_gpu_manufacturer 3": 52, - "instance_gpu_memory 1": 16, - "instance_gpu_memory 2": 33, - "instance_gpu_memory 3": 53, - "instance_gpu_name 1": 17, - "instance_gpu_name 2": 34, - "instance_gpu_name 3": 54, - "instance_hypervisor 1": 18, - "instance_hypervisor 2": 35, - "instance_hypervisor 3": 55, - "instance_memory 1": 19, - "instance_memory 2": 36, - "instance_memory 3": 56, - "instance_pods 1": 20, - "instance_pods 2": 37, - "instance_pods 3": 57, - "instance_size 1": 21, - "instance_size 2": 38, - "instance_size 3": 58, - "instance_type 1": 3, - "instance_type 2": 39, - "instance_type 3": 59, - "job 1": 22, - "job 2": 40, - "job 3": 60, - "node": 66, - "node_name": 0, - "os 1": 23, - "os 2": 41, - "os 3": 61, - "nodepool 1": 2, - "nodepool 2": 42, - "nodepool 3": 62, - "resource_type 1": 24, - "resource_type 2": 43, - "resource_type 3": 63, - "zone 1": 7, - "zone 2": 44, - "zone 3": 64 - }, - "renameByName": { - "Time": "", - "Time 1": "", - "Value": "CPU Utilization", - "Value #Allocatable": "", - "Value #CPU Utilization": "CPU Utilization", - "Value #Memory Utilization": "Memory Utilization", - "Value #Pod CPU": "", - "Value #Pod Count": "Pods", - "Value #Uptime": "Uptime", - "arch": "Architecture", - "arch 1": "Arch", - "capacity_type": "Capacity Type", - "capacity_type 1": "Capacity Type", - "instance 1": "Instance", - "instance_cpu 1": "vCPU", - "instance_type": "Instance Type", - "instance_type 1": "Instance Type", - "node_name": "Node Name", - "nodepool 1": "Nodepool", - "zone 1": "Zone" - } - } - } - ], - "type": "table" - } - ], - "refresh": false, - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "includeAll": false, - "label": "Data Source", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, nodepool)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "nodepool", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, nodepool)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, zone)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "zone", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, zone)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, arch)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "arch", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, arch)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, capacity_type)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "capacity_type", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, capacity_type)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, instance_type)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "instance_type", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, instance_type)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": "nodepool", - "value": "nodepool" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "distribution_filter", - "options": [ - { - "selected": false, - "text": "arch", - "value": "arch" - }, - { - "selected": false, - "text": "capacity_type", - "value": "capacity_type" - }, - { - "selected": false, - "text": "instance_type", - "value": "instance_type" - }, - { - "selected": false, - "text": "namespace", - "value": "namespace" - }, - { - "selected": false, - "text": "node", - "value": "node" - }, - { - "selected": true, - "text": "nodepool", - "value": "nodepool" - }, - { - "selected": false, - "text": "zone", - "value": "zone" - } - ], - "query": "arch,capacity_type,instance_type,namespace,node,nodepool,zone", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Karpenter Capacity", - "uid": "ta8I9Q67z", - "version": 4, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 2, + "id": 44, + "links": [], + "liveNow": true, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(cluster,nodepool) (karpenter_nodes_created_total{nodepool=~\"$nodepool\"})", + "format": "time_series", + "legendFormat": "{{cluster}}", + "range": true, + "refId": "A" + } + ], + "title": "Nodes Created: nodepool \"$nodepool\"", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(cluster,nodepool) (karpenter_nodes_terminated_total{nodepool=~\"$nodepool\"})", + "format": "time_series", + "legendFormat": "{{cluster}}", + "range": true, + "refId": "A" + } + ], + "title": "Nodes Terminated: nodepool \"$nodepool\"", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(consolidation_type,method)(karpenter_disruption_eligible_nodes)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Nodes Eligible for Disruptions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(action,consolidation_type,method)(karpenter_disruption_decisions_total)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Disruption Actions Performed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pod Distribution: $distribution_filter", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(phase)(karpenter_pods_state)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pod Phase", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*Utilization$" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 1 + }, + { + "id": "unit", + "value": "percentunit" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Provisioned" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 18, + "x": 0, + "y": 42 + }, + "id": 10, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "karpenter_nodepool_usage{resource_type=\"cpu\"} / karpenter_nodepool_limit{resource_type=\"cpu\"}", + "format": "table", + "instant": true, + "legendFormat": "CPU Limit Utilization", + "range": false, + "refId": "CPU Limit Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "Node Count" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "karpenter_nodepool_usage{resource_type=\"memory\"} / karpenter_nodepool_limit{resource_type=\"memory\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Memory Limit Utilization", + "range": false, + "refId": "Memory Limit Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "CPU Capacity" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "Memory Capacity" + } + ], + "title": "Nodepool Summary", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "nodepool" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "__name__": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "job": true, + "job 1": true, + "job 2": true, + "resource_type": true, + "resource_type 1": true, + "resource_type 2": true + }, + "indexByName": { + "Time 1": 6, + "Time 2": 7, + "Time 3": 11, + "Time 4": 15, + "Time 5": 16, + "Value #CPU Capacity": 2, + "Value #CPU Limit Utilization": 3, + "Value #Memory Capacity": 4, + "Value #Memory Limit Utilization": 5, + "Value #Node Count": 1, + "instance 1": 8, + "instance 2": 12, + "job 1": 9, + "job 2": 13, + "nodepool": 0, + "resource_type 1": 10, + "resource_type 2": 14 + }, + "renameByName": { + "Time 1": "", + "Value": "CPU Utilization", + "Value #CPU Capacity": "CPU Provisioned", + "Value #CPU Limit Utilization": "CPU Limit Utilization", + "Value #CPU Utilization": "CPU Limit Utilization", + "Value #Memory Capacity": "Memory Provisioned", + "Value #Memory Limit Utilization": "Memory Limit Utilization", + "Value #Memory Utilization": "Memory Utilization", + "Value #Node Count": "Node Count", + "instance": "", + "instance 1": "", + "job": "", + "nodepool": "Nodepool" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 18, + "y": 42 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", + "legendFormat": "Percentage", + "range": true, + "refId": "A" + } + ], + "title": "Spot Node Percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "node_name" + }, + "properties": [ + { + "id": "custom.width", + "value": 333 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Utilization" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "percentage", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 75 + } + ] + } + }, + { + "id": "max", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Uptime" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "decimals", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Uptime" + } + ] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "CPU Utilization", + "range": false, + "refId": "CPU Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Memory Utilization", + "range": false, + "refId": "Memory Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Memory Utilization", + "range": false, + "refId": "Pod Count" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\n sum by (node)(node_time_seconds) - sum by (node)(node_boot_time_seconds),\n \"node_name\", \"$1\", \"node\", \"(.+)\"\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Uptime", + "range": false, + "refId": "Uptime" + } + ], + "title": "Node Summary", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "node_name" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Value": false, + "Value #Pod Count": false, + "__name__": true, + "arch": true, + "arch 1": true, + "arch 2": true, + "arch 3": true, + "capacity_type 2": true, + "capacity_type 3": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "instance 3": true, + "instance_category 1": true, + "instance_category 2": true, + "instance_category 3": true, + "instance_cpu": true, + "instance_cpu 1": true, + "instance_cpu 2": true, + "instance_cpu 3": true, + "instance_family": true, + "instance_family 1": true, + "instance_family 2": true, + "instance_family 3": true, + "instance_generation 1": true, + "instance_generation 2": true, + "instance_generation 3": true, + "instance_gpu_count": true, + "instance_gpu_count 1": true, + "instance_gpu_count 2": true, + "instance_gpu_count 3": true, + "instance_gpu_manufacturer": true, + "instance_gpu_manufacturer 1": true, + "instance_gpu_manufacturer 2": true, + "instance_gpu_manufacturer 3": true, + "instance_gpu_memory": true, + "instance_gpu_memory 1": true, + "instance_gpu_memory 2": true, + "instance_gpu_memory 3": true, + "instance_gpu_name": true, + "instance_gpu_name 1": true, + "instance_gpu_name 2": true, + "instance_gpu_name 3": true, + "instance_hypervisor": true, + "instance_hypervisor 1": true, + "instance_hypervisor 2": true, + "instance_hypervisor 3": true, + "instance_local_nvme 1": true, + "instance_local_nvme 2": true, + "instance_local_nvme 3": true, + "instance_memory": true, + "instance_memory 1": true, + "instance_memory 2": true, + "instance_memory 3": true, + "instance_pods": true, + "instance_pods 1": true, + "instance_pods 2": true, + "instance_pods 3": true, + "instance_size": true, + "instance_size 1": true, + "instance_size 2": true, + "instance_size 3": true, + "instance_type 1": false, + "instance_type 2": true, + "instance_type 3": true, + "job": true, + "job 1": true, + "job 2": true, + "job 3": true, + "node": true, + "nodepool 1": false, + "nodepool 2": true, + "nodepool 3": true, + "os": true, + "os 1": true, + "os 2": true, + "os 3": true, + "resource_type": true, + "resource_type 1": true, + "resource_type 2": true, + "resource_type 3": true, + "zone 1": false, + "zone 2": true, + "zone 3": true + }, + "indexByName": { + "Time 1": 1, + "Time 2": 25, + "Time 3": 45, + "Time 4": 65, + "Value #CPU Utilization": 10, + "Value #Memory Utilization": 11, + "Value #Pod Count": 9, + "Value #Uptime": 8, + "arch 1": 5, + "arch 2": 26, + "arch 3": 46, + "capacity_type 1": 6, + "capacity_type 2": 27, + "capacity_type 3": 47, + "instance 1": 4, + "instance 2": 28, + "instance 3": 48, + "instance_cpu 1": 12, + "instance_cpu 2": 29, + "instance_cpu 3": 49, + "instance_family 1": 13, + "instance_family 2": 30, + "instance_family 3": 50, + "instance_gpu_count 1": 14, + "instance_gpu_count 2": 31, + "instance_gpu_count 3": 51, + "instance_gpu_manufacturer 1": 15, + "instance_gpu_manufacturer 2": 32, + "instance_gpu_manufacturer 3": 52, + "instance_gpu_memory 1": 16, + "instance_gpu_memory 2": 33, + "instance_gpu_memory 3": 53, + "instance_gpu_name 1": 17, + "instance_gpu_name 2": 34, + "instance_gpu_name 3": 54, + "instance_hypervisor 1": 18, + "instance_hypervisor 2": 35, + "instance_hypervisor 3": 55, + "instance_memory 1": 19, + "instance_memory 2": 36, + "instance_memory 3": 56, + "instance_pods 1": 20, + "instance_pods 2": 37, + "instance_pods 3": 57, + "instance_size 1": 21, + "instance_size 2": 38, + "instance_size 3": 58, + "instance_type 1": 3, + "instance_type 2": 39, + "instance_type 3": 59, + "job 1": 22, + "job 2": 40, + "job 3": 60, + "node": 66, + "node_name": 0, + "nodepool 1": 2, + "nodepool 2": 42, + "nodepool 3": 62, + "os 1": 23, + "os 2": 41, + "os 3": 61, + "resource_type 1": 24, + "resource_type 2": 43, + "resource_type 3": 63, + "zone 1": 7, + "zone 2": 44, + "zone 3": 64 + }, + "renameByName": { + "Time": "", + "Time 1": "", + "Value": "CPU Utilization", + "Value #Allocatable": "", + "Value #CPU Utilization": "CPU Utilization", + "Value #Memory Utilization": "Memory Utilization", + "Value #Pod CPU": "", + "Value #Pod Count": "Pods", + "Value #Uptime": "Uptime", + "arch": "Architecture", + "arch 1": "Arch", + "capacity_type": "Capacity Type", + "capacity_type 1": "Capacity Type", + "instance 1": "Instance", + "instance_cpu 1": "vCPU", + "instance_type": "Instance Type", + "instance_type 1": "Instance Type", + "node_name": "Node Name", + "nodepool 1": "Nodepool", + "zone 1": "Zone" + } + } + } + ], + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, nodepool)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "nodepool", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, nodepool)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, zone)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "zone", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, zone)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, arch)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "arch", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, arch)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, capacity_type)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "capacity_type", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, capacity_type)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, instance_type)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "instance_type", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, instance_type)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(karpenter_disruption_decisions_total,method)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(karpenter_disruption_decisions_total,method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "nodepool", + "value": "nodepool" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "distribution_filter", + "options": [ + { + "selected": false, + "text": "arch", + "value": "arch" + }, + { + "selected": false, + "text": "capacity_type", + "value": "capacity_type" + }, + { + "selected": false, + "text": "instance_type", + "value": "instance_type" + }, + { + "selected": false, + "text": "method", + "value": "method" + }, + { + "selected": false, + "text": "namespace", + "value": "namespace" + }, + { + "selected": false, + "text": "node", + "value": "node" + }, + { + "selected": true, + "text": "nodepool", + "value": "nodepool" + }, + { + "selected": false, + "text": "zone", + "value": "zone" + } + ], + "query": "arch,capacity_type,instance_type,method,namespace,node,nodepool,zone", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Karpenter Capacity", + "uid": "ta8I9Q67z", + "version": 5, + "weekStart": "" } diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json b/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json index c7762d302f99..174282ee9468 100644 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json @@ -498,7 +498,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(controller_runtime_reconcile_total[10m])) by (controller)", + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"karpenter\"}[10m])) by (controller)", "legendFormat": "{{controller}}", "range": true, "refId": "A" @@ -542,14 +542,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(controller_runtime_reconcile_time_seconds_count, controller)", + "definition": "label_values(controller_runtime_reconcile_time_seconds_count{job=\"karpenter\"}, controller)", "hide": 0, "includeAll": false, "multi": false, "name": "controller", "options": [], "query": { - "query": "label_values(controller_runtime_reconcile_time_seconds_count, controller)", + "query": "label_values(controller_runtime_reconcile_time_seconds_count{job=\"karpenter\"}, controller)", "refId": "StandardVariableQuery" }, "refresh": 2, diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh index fa577d724e9f..73643e64bd65 100755 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh @@ -1,4 +1,4 @@ -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > $TEMPOUT \ +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/main/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > $TEMPOUT \ && aws cloudformation deploy \ --stack-name "Karpenter-${CLUSTER_NAME}" \ --template-file "${TEMPOUT}" \ diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh index 0ab6f5f464bd..3c8b34bc0f94 100755 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh @@ -1,4 +1,4 @@ -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > "${TEMPOUT}" \ +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/main/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > "${TEMPOUT}" \ && aws cloudformation deploy \ --stack-name "Karpenter-${CLUSTER_NAME}" \ --template-file "${TEMPOUT}" \ diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh index 54e826db269b..f7ee31c2cd8f 100755 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh @@ -1,6 +1,6 @@ TEMPOUT="$(mktemp)" -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > "${TEMPOUT}" \ +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/main/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > "${TEMPOUT}" \ && aws cloudformation deploy \ --stack-name "Karpenter-${CLUSTER_NAME}" \ --template-file "${TEMPOUT}" \ diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh index 33f1cb553b1b..f3625e936810 100755 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh @@ -1,5 +1,5 @@ cat < AWS instance types offer varying resources and can be selected by labels. The values provided below are the resources available with some assumptions and after the instance overhead has been subtracted: - `blockDeviceMappings` are not configured -- `aws-eni-limited-pod-density` is assumed to be `true` -- `amiFamily` is set to the default of `AL2` +- `amiFamily` is set to `AL2023` ## a1 Family ### `a1.medium` #### Labels @@ -20,6 +19,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -45,6 +45,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -70,6 +71,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -95,6 +97,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -120,6 +123,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -145,6 +149,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -335,6 +340,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -358,6 +364,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -381,6 +388,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -404,6 +412,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -427,6 +436,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|36| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -452,6 +462,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -477,6 +488,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -502,6 +514,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -527,6 +540,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -552,6 +566,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|36| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -577,6 +592,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -602,6 +618,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|72| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -627,6 +644,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -652,6 +670,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -678,6 +697,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -703,6 +723,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -728,6 +749,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -753,6 +775,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -778,6 +801,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -803,6 +827,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -828,6 +853,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6300| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -853,6 +879,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -879,6 +906,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -905,6 +933,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -931,6 +960,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -957,6 +987,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -983,6 +1014,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -1009,6 +1041,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -1035,6 +1068,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6300| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -1061,6 +1095,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -1088,6 +1123,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1114,6 +1150,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1140,6 +1177,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1166,6 +1204,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1192,6 +1231,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|36| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1218,6 +1258,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1244,6 +1285,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|72| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1270,6 +1312,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1296,6 +1339,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1323,6 +1367,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1348,6 +1393,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1373,6 +1419,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1398,6 +1445,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1423,6 +1471,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|36| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1449,6 +1498,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|72| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1475,6 +1525,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|72| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1502,6 +1553,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1527,6 +1579,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1552,6 +1605,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1577,6 +1631,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1602,6 +1657,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1627,6 +1683,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1652,6 +1709,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1677,6 +1735,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1702,6 +1761,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1727,6 +1787,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1753,6 +1814,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1780,6 +1842,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1805,6 +1868,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1830,6 +1894,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1855,6 +1920,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1880,6 +1946,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1905,6 +1972,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1930,6 +1998,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1955,6 +2024,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1980,6 +2050,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -2006,6 +2077,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2032,6 +2104,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2058,6 +2131,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2084,6 +2158,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2110,6 +2185,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2136,6 +2212,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2162,6 +2239,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2188,6 +2266,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2214,6 +2293,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2241,6 +2321,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2266,6 +2347,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2291,6 +2373,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2316,6 +2399,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2341,6 +2425,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2366,6 +2451,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2391,6 +2477,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|28500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2416,6 +2503,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2443,6 +2531,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2468,6 +2557,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2493,6 +2583,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2518,6 +2609,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2543,6 +2635,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2568,6 +2661,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2593,6 +2687,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2618,6 +2713,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2643,6 +2739,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2669,6 +2766,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2696,6 +2794,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2722,6 +2821,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2748,6 +2848,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2774,6 +2875,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2800,6 +2902,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2826,6 +2929,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2852,6 +2956,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2878,6 +2983,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2904,6 +3010,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2931,6 +3038,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2959,6 +3067,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -2984,6 +3093,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3009,6 +3119,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3034,6 +3145,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3059,6 +3171,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3084,6 +3197,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|37500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3109,6 +3223,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|50000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3134,6 +3249,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|75000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3159,6 +3275,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3174,10 +3291,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|238333Mi| - |pods|345| + |memory|237794Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ### `c6in.metal` #### Labels | Label | Value | @@ -3185,6 +3302,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3200,10 +3318,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|238333Mi| - |pods|345| + |memory|237794Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ## c7a Family ### `c7a.medium` #### Labels @@ -3212,6 +3330,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3237,6 +3356,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3262,6 +3382,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3287,6 +3408,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3312,6 +3434,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3337,6 +3460,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3362,6 +3486,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3387,6 +3512,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3412,6 +3538,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3437,6 +3564,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3462,6 +3590,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3488,6 +3617,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| @@ -3515,6 +3645,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3540,6 +3671,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3565,6 +3697,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3590,6 +3723,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3615,6 +3749,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3640,6 +3775,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3665,6 +3801,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3690,6 +3827,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3716,6 +3854,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3735,6 +3874,151 @@ below are the resources available with some assumptions and after the instance o |pods|737| |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| +## c7g-flex Family +### `c7g-flex.medium` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7g-flex| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|2048| + |karpenter.k8s.aws/instance-size|medium| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7g-flex.medium| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|940m| + |ephemeral-storage|17Gi| + |memory|1392Mi| + |pods|8| +### `c7g-flex.large` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7g-flex| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|4096| + |karpenter.k8s.aws/instance-size|large| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7g-flex.large| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|1930m| + |ephemeral-storage|17Gi| + |memory|3055Mi| + |pods|29| +### `c7g-flex.xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7g-flex| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|8192| + |karpenter.k8s.aws/instance-size|xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7g-flex.xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|3920m| + |ephemeral-storage|17Gi| + |memory|6525Mi| + |pods|58| +### `c7g-flex.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7g-flex| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7g-flex.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|14103Mi| + |pods|58| +### `c7g-flex.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7g-flex| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7g-flex.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|27322Mi| + |pods|234| +### `c7g-flex.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7g-flex| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7g-flex.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|57632Mi| + |pods|234| ## c7gd Family ### `c7gd.medium` #### Labels @@ -3743,6 +4027,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3769,6 +4054,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3795,6 +4081,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3821,6 +4108,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3847,6 +4135,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3873,6 +4162,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3899,6 +4189,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3925,6 +4216,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3952,6 +4244,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3971,6 +4264,7 @@ below are the resources available with some assumptions and after the instance o |memory|112720Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## c7gn Family ### `c7gn.medium` #### Labels @@ -3979,6 +4273,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4004,6 +4299,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4029,6 +4325,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4054,6 +4351,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4079,6 +4377,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4104,6 +4403,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4129,6 +4429,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4154,6 +4455,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -4180,11 +4482,13 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|200000| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -4197,6 +4501,7 @@ below are the resources available with some assumptions and after the instance o |memory|112720Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## c7i Family ### `c7i.large` #### Labels @@ -4205,6 +4510,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4230,6 +4536,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4255,6 +4562,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4280,6 +4588,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4305,6 +4614,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4330,6 +4640,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4355,6 +4666,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4380,6 +4692,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4405,6 +4718,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4430,6 +4744,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4456,6 +4771,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4483,6 +4799,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|d2| |karpenter.k8s.aws/instance-generation|2| @@ -4506,6 +4823,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|d2| |karpenter.k8s.aws/instance-generation|2| @@ -4529,6 +4847,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|d2| |karpenter.k8s.aws/instance-generation|2| @@ -4552,6 +4871,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|36| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|d2| |karpenter.k8s.aws/instance-generation|2| @@ -4577,6 +4897,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3| |karpenter.k8s.aws/instance-generation|3| @@ -4603,6 +4924,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3| |karpenter.k8s.aws/instance-generation|3| @@ -4629,6 +4951,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3| |karpenter.k8s.aws/instance-generation|3| @@ -4655,6 +4978,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|5000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3| |karpenter.k8s.aws/instance-generation|3| @@ -4682,6 +5006,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4708,6 +5033,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4734,6 +5060,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4760,6 +5087,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4786,6 +5114,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|5000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4812,6 +5141,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4839,6 +5169,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|dl| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|dl1| |karpenter.k8s.aws/instance-generation|1| @@ -4872,13 +5203,13 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|f| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1700| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|f1| |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-local-nvme|470| |karpenter.k8s.aws/instance-memory|124928| - |karpenter.k8s.aws/instance-network-bandwidth|2500| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4897,13 +5228,13 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|f| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|f1| |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-local-nvme|940| |karpenter.k8s.aws/instance-memory|249856| - |karpenter.k8s.aws/instance-network-bandwidth|5000| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4922,6 +5253,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|f| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|f1| |karpenter.k8s.aws/instance-generation|1| @@ -4948,6 +5280,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g3| |karpenter.k8s.aws/instance-generation|3| @@ -4957,7 +5290,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-gpu-name|m60| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|124928| - |karpenter.k8s.aws/instance-network-bandwidth|5000| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4970,6 +5302,7 @@ below are the resources available with some assumptions and after the instance o |memory|112629Mi| |nvidia.com/gpu|1| |pods|234| + |vpc.amazonaws.com/pod-eni|6| ### `g3.8xlarge` #### Labels | Label | Value | @@ -4977,6 +5310,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g3| |karpenter.k8s.aws/instance-generation|3| @@ -4999,6 +5333,7 @@ below are the resources available with some assumptions and after the instance o |memory|228187Mi| |nvidia.com/gpu|2| |pods|234| + |vpc.amazonaws.com/pod-eni|6| ### `g3.16xlarge` #### Labels | Label | Value | @@ -5006,6 +5341,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g3| |karpenter.k8s.aws/instance-generation|3| @@ -5036,6 +5372,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|850| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g3s| |karpenter.k8s.aws/instance-generation|3| @@ -5057,6 +5394,7 @@ below are the resources available with some assumptions and after the instance o |memory|27896Mi| |nvidia.com/gpu|1| |pods|58| + |vpc.amazonaws.com/pod-eni|10| ## g4ad Family ### `g4ad.xlarge` #### Labels @@ -5065,6 +5403,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -5096,6 +5435,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -5127,6 +5467,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -5158,6 +5499,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -5189,6 +5531,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6300| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -5221,6 +5564,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5252,6 +5596,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5283,6 +5628,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5314,6 +5660,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5346,6 +5693,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5378,6 +5726,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5410,6 +5759,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5443,6 +5793,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5474,6 +5825,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5505,6 +5857,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5536,6 +5889,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|16000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5568,6 +5922,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|16000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5600,6 +5955,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|16000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5632,6 +5988,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5664,6 +6021,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5697,6 +6055,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5727,6 +6086,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5757,6 +6117,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5787,6 +6148,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5817,6 +6179,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5847,6 +6210,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5878,6 +6242,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|5000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -5888,6 +6253,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|250| |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|2500| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5900,6 +6266,7 @@ below are the resources available with some assumptions and after the instance o |memory|14162Mi| |nvidia.com/gpu|1| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `g6.2xlarge` #### Labels | Label | Value | @@ -5907,6 +6274,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|5000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -5917,6 +6285,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|450| |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|5000| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5929,6 +6298,7 @@ below are the resources available with some assumptions and after the instance o |memory|29317Mi| |nvidia.com/gpu|1| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `g6.4xlarge` #### Labels | Label | Value | @@ -5936,6 +6306,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|8000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -5946,6 +6317,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|10000| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5958,6 +6330,7 @@ below are the resources available with some assumptions and after the instance o |memory|57691Mi| |nvidia.com/gpu|1| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `g6.8xlarge` #### Labels | Label | Value | @@ -5965,6 +6338,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|16000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -5975,6 +6349,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5988,6 +6363,7 @@ below are the resources available with some assumptions and after the instance o |nvidia.com/gpu|1| |pods|234| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|84| ### `g6.12xlarge` #### Labels | Label | Value | @@ -5995,6 +6371,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -6003,8 +6380,9 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-gpu-memory|91553| |karpenter.k8s.aws/instance-gpu-name|l4| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|15200| + |karpenter.k8s.aws/instance-local-nvme|3760| |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|40000| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6018,6 +6396,7 @@ below are the resources available with some assumptions and after the instance o |nvidia.com/gpu|4| |pods|234| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|114| ### `g6.16xlarge` #### Labels | Label | Value | @@ -6025,6 +6404,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -6033,8 +6413,9 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-gpu-memory|22888| |karpenter.k8s.aws/instance-gpu-name|l4| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|3800| + |karpenter.k8s.aws/instance-local-nvme|1880| |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6048,6 +6429,7 @@ below are the resources available with some assumptions and after the instance o |nvidia.com/gpu|1| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `g6.24xlarge` #### Labels | Label | Value | @@ -6055,6 +6437,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -6063,8 +6446,9 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-gpu-memory|91553| |karpenter.k8s.aws/instance-gpu-name|l4| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|15200| + |karpenter.k8s.aws/instance-local-nvme|3760| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6078,6 +6462,7 @@ below are the resources available with some assumptions and after the instance o |nvidia.com/gpu|4| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `g6.48xlarge` #### Labels | Label | Value | @@ -6085,6 +6470,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g6| |karpenter.k8s.aws/instance-generation|6| @@ -6093,8 +6479,9 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-gpu-memory|183105| |karpenter.k8s.aws/instance-gpu-name|l4| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|60800| + |karpenter.k8s.aws/instance-local-nvme|7520| |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|100000| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6108,6 +6495,7 @@ below are the resources available with some assumptions and after the instance o |nvidia.com/gpu|8| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## gr6 Family ### `gr6.4xlarge` #### Labels @@ -6116,6 +6504,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|gr| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|8000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|gr6| |karpenter.k8s.aws/instance-generation|6| @@ -6126,6 +6515,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|10000| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6138,6 +6528,7 @@ below are the resources available with some assumptions and after the instance o |memory|118312Mi| |nvidia.com/gpu|1| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `gr6.8xlarge` #### Labels | Label | Value | @@ -6145,6 +6536,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|gr| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|16000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|gr6| |karpenter.k8s.aws/instance-generation|6| @@ -6155,6 +6547,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6168,6 +6561,7 @@ below are the resources available with some assumptions and after the instance o |nvidia.com/gpu|1| |pods|234| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|84| ## h1 Family ### `h1.2xlarge` #### Labels @@ -6176,6 +6570,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|h| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|h1| |karpenter.k8s.aws/instance-generation|1| @@ -6200,6 +6595,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|h| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|h1| |karpenter.k8s.aws/instance-generation|1| @@ -6224,6 +6620,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|h| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|h1| |karpenter.k8s.aws/instance-generation|1| @@ -6248,6 +6645,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|h| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|h1| |karpenter.k8s.aws/instance-generation|1| @@ -6273,6 +6671,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|hpc| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|hpc7g| |karpenter.k8s.aws/instance-generation|7| @@ -6298,6 +6697,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|hpc| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|hpc7g| |karpenter.k8s.aws/instance-generation|7| @@ -6323,6 +6723,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|hpc| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|hpc7g| |karpenter.k8s.aws/instance-generation|7| @@ -6443,6 +6844,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|425| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6468,6 +6870,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|850| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6493,6 +6896,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1700| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6518,6 +6922,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6543,6 +6948,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6568,6 +6974,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6593,6 +7000,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|72| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -6620,6 +7028,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6646,6 +7055,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6672,6 +7082,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6698,6 +7109,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|12| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6724,6 +7136,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6750,6 +7163,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6777,6 +7191,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6804,6 +7219,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6832,6 +7248,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6858,6 +7275,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6884,6 +7302,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6910,6 +7329,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6936,6 +7356,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6962,6 +7383,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6990,6 +7412,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7015,6 +7438,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7041,6 +7465,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7067,6 +7492,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7093,6 +7519,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7119,6 +7546,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7145,6 +7573,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7171,6 +7600,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7197,6 +7627,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7224,6 +7655,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -7252,6 +7684,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -7278,6 +7711,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -7304,6 +7738,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -7330,6 +7765,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -7356,6 +7792,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -7382,6 +7819,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -7413,6 +7851,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf1| |karpenter.k8s.aws/instance-generation|1| @@ -7442,6 +7881,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf1| |karpenter.k8s.aws/instance-generation|1| @@ -7471,6 +7911,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf1| |karpenter.k8s.aws/instance-generation|1| @@ -7500,6 +7941,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf1| |karpenter.k8s.aws/instance-generation|1| @@ -7531,6 +7973,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf2| |karpenter.k8s.aws/instance-generation|2| @@ -7560,6 +8003,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf2| |karpenter.k8s.aws/instance-generation|2| @@ -7589,6 +8033,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf2| |karpenter.k8s.aws/instance-generation|2| @@ -7618,6 +8063,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf2| |karpenter.k8s.aws/instance-generation|2| @@ -7645,6 +8091,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7671,6 +8118,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7697,6 +8145,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7723,6 +8172,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7749,6 +8199,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7775,6 +8226,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -8058,6 +8510,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|450| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -8081,6 +8534,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -8104,6 +8558,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -8127,6 +8582,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -8150,6 +8606,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|40| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -8174,6 +8631,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -8199,6 +8657,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8224,6 +8683,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8249,6 +8709,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8274,6 +8735,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8299,6 +8761,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8324,6 +8787,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8349,6 +8813,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8374,6 +8839,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8399,6 +8865,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -8425,6 +8892,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8450,6 +8918,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8475,6 +8944,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8500,6 +8970,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8525,6 +8996,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8550,6 +9022,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8575,6 +9048,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8600,6 +9074,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|13750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -8626,6 +9101,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8652,6 +9128,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8678,6 +9155,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8704,6 +9182,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8730,6 +9209,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8756,6 +9236,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8782,6 +9263,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8808,6 +9290,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|13750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8835,6 +9318,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8861,6 +9345,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8887,6 +9372,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8913,6 +9399,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8939,6 +9426,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8965,6 +9453,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8991,6 +9480,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -9017,6 +9507,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -9043,6 +9534,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -9070,6 +9562,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9096,6 +9589,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9122,6 +9616,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9148,6 +9643,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9174,6 +9670,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9200,6 +9697,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9226,6 +9724,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9252,6 +9751,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9279,6 +9779,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -9307,6 +9808,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9332,6 +9834,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9357,6 +9860,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9382,6 +9886,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9407,6 +9912,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9432,6 +9938,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9457,6 +9964,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9482,6 +9990,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9508,6 +10017,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -9535,6 +10045,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9560,6 +10071,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9585,6 +10097,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9610,6 +10123,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|12| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9635,6 +10149,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9660,6 +10175,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9686,6 +10202,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -9713,6 +10230,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9738,6 +10256,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9763,6 +10282,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9788,6 +10308,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9813,6 +10334,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9838,6 +10360,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9863,6 +10386,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9888,6 +10412,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9913,6 +10438,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9938,6 +10464,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9964,6 +10491,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9991,6 +10519,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10016,6 +10545,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10041,6 +10571,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10066,6 +10597,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10091,6 +10623,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10116,6 +10649,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10141,6 +10675,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10166,6 +10701,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10191,6 +10727,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -10217,6 +10754,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10243,6 +10781,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10269,6 +10808,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10295,6 +10835,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10321,6 +10862,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10347,6 +10889,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10373,6 +10916,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10399,6 +10943,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10425,6 +10970,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -10452,6 +10998,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10477,6 +11024,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10502,6 +11050,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10527,6 +11076,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10552,6 +11102,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10577,6 +11128,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10602,6 +11154,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10627,6 +11180,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10652,6 +11206,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10678,6 +11233,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -10705,6 +11261,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10731,6 +11288,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10757,6 +11315,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10783,6 +11342,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10809,6 +11369,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10835,6 +11396,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10861,6 +11423,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10887,6 +11450,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10913,6 +11477,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10940,6 +11505,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10968,6 +11534,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -10994,6 +11561,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11020,6 +11588,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11046,6 +11615,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11072,6 +11642,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11098,6 +11669,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|37500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11124,6 +11696,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|50000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11150,6 +11723,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|75000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11176,6 +11750,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11192,10 +11767,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|480816Mi| - |pods|345| + |memory|480277Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ### `m6idn.metal` #### Labels | Label | Value | @@ -11203,6 +11778,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -11219,10 +11795,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|480816Mi| - |pods|345| + |memory|480277Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ## m6in Family ### `m6in.large` #### Labels @@ -11231,6 +11807,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11256,6 +11833,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11281,6 +11859,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11306,6 +11885,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11331,6 +11911,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11356,6 +11937,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|37500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11381,6 +11963,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|50000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11406,6 +11989,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|75000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11431,6 +12015,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11446,10 +12031,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|480816Mi| - |pods|345| + |memory|480277Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ### `m6in.metal` #### Labels | Label | Value | @@ -11457,6 +12042,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -11472,10 +12058,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|480816Mi| - |pods|345| + |memory|480277Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ## m7a Family ### `m7a.medium` #### Labels @@ -11484,6 +12070,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11509,6 +12096,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11534,6 +12122,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11559,6 +12148,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11584,6 +12174,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11609,6 +12200,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11634,6 +12226,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11659,6 +12252,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11684,6 +12278,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11709,6 +12304,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11734,6 +12330,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11760,6 +12357,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -11787,6 +12385,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11812,6 +12411,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11837,6 +12437,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11862,6 +12463,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11887,6 +12489,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11912,6 +12515,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11937,6 +12541,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11962,6 +12567,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11988,6 +12594,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -12007,25 +12614,24 @@ below are the resources available with some assumptions and after the instance o |pods|737| |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| -## m7gd Family -### `m7gd.medium` +## m7g-flex Family +### `m7g-flex.medium` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-family|m7g-flex| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|4096| - |karpenter.k8s.aws/instance-network-bandwidth|520| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|m7gd.medium| + |node.kubernetes.io/instance-type|m7g-flex.medium| #### Resources | Resource | Quantity | |--|--| @@ -12033,25 +12639,23 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|3286Mi| |pods|8| - |vpc.amazonaws.com/pod-eni|4| -### `m7gd.large` +### `m7g-flex.large` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-family|m7g-flex| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|8192| - |karpenter.k8s.aws/instance-network-bandwidth|937| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|m7gd.large| + |node.kubernetes.io/instance-type|m7g-flex.large| #### Resources | Resource | Quantity | |--|--| @@ -12059,25 +12663,23 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|6844Mi| |pods|29| - |vpc.amazonaws.com/pod-eni|9| -### `m7gd.xlarge` +### `m7g-flex.xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-family|m7g-flex| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|16384| - |karpenter.k8s.aws/instance-network-bandwidth|1876| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|m7gd.xlarge| + |node.kubernetes.io/instance-type|m7g-flex.xlarge| #### Resources | Resource | Quantity | |--|--| @@ -12085,25 +12687,23 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14103Mi| |pods|58| - |vpc.amazonaws.com/pod-eni|18| -### `m7gd.2xlarge` +### `m7g-flex.2xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-family|m7g-flex| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|32768| - |karpenter.k8s.aws/instance-network-bandwidth|3750| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|m7gd.2xlarge| + |node.kubernetes.io/instance-type|m7g-flex.2xlarge| #### Resources | Resource | Quantity | |--|--| @@ -12111,25 +12711,23 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|29258Mi| |pods|58| - |vpc.amazonaws.com/pod-eni|38| -### `m7gd.4xlarge` +### `m7g-flex.4xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-family|m7g-flex| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|65536| - |karpenter.k8s.aws/instance-network-bandwidth|7500| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|m7gd.4xlarge| + |node.kubernetes.io/instance-type|m7g-flex.4xlarge| #### Resources | Resource | Quantity | |--|--| @@ -12137,25 +12735,23 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|57632Mi| |pods|234| - |vpc.amazonaws.com/pod-eni|54| -### `m7gd.8xlarge` +### `m7g-flex.8xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-family|m7g-flex| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|131072| - |karpenter.k8s.aws/instance-network-bandwidth|15000| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|m7gd.8xlarge| + |node.kubernetes.io/instance-type|m7g-flex.8xlarge| #### Resources | Resource | Quantity | |--|--| @@ -12163,71 +12759,236 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118253Mi| |pods|234| - |vpc.amazonaws.com/pod-eni|54| -### `m7gd.12xlarge` +## m7gd Family +### `m7gd.medium` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|m| - |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|2850| - |karpenter.k8s.aws/instance-memory|196608| - |karpenter.k8s.aws/instance-network-bandwidth|22500| - |karpenter.k8s.aws/instance-size|12xlarge| + |karpenter.k8s.aws/instance-local-nvme|59| + |karpenter.k8s.aws/instance-memory|4096| + |karpenter.k8s.aws/instance-network-bandwidth|520| + |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|m7gd.12xlarge| + |node.kubernetes.io/instance-type|m7gd.medium| #### Resources | Resource | Quantity | |--|--| - |cpu|47810m| + |cpu|940m| |ephemeral-storage|17Gi| - |memory|178874Mi| - |pods|234| - |vpc.amazonaws.com/pod-eni|54| -### `m7gd.16xlarge` + |memory|3286Mi| + |pods|8| + |vpc.amazonaws.com/pod-eni|4| +### `m7gd.large` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|m| - |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|3800| - |karpenter.k8s.aws/instance-memory|262144| - |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-size|16xlarge| + |karpenter.k8s.aws/instance-local-nvme|118| + |karpenter.k8s.aws/instance-memory|8192| + |karpenter.k8s.aws/instance-network-bandwidth|937| + |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|m7gd.16xlarge| + |node.kubernetes.io/instance-type|m7gd.large| #### Resources | Resource | Quantity | |--|--| - |cpu|63770m| + |cpu|1930m| |ephemeral-storage|17Gi| - |memory|233962Mi| - |pods|737| - |vpc.amazonaws.com/efa|1| - |vpc.amazonaws.com/pod-eni|107| -### `m7gd.metal` + |memory|6844Mi| + |pods|29| + |vpc.amazonaws.com/pod-eni|9| +### `m7gd.xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|m| - |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| - |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|237| + |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|1876| + |karpenter.k8s.aws/instance-size|xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|m7gd.xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|3920m| + |ephemeral-storage|17Gi| + |memory|14103Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|18| +### `m7gd.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|m| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|474| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|3750| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|m7gd.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|29258Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|38| +### `m7gd.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|m| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|950| + |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|7500| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|m7gd.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|57632Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `m7gd.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|m| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|1900| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|15000| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|m7gd.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|118253Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `m7gd.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|m| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|2850| + |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|22500| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|m7gd.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|178874Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `m7gd.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|m| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|3800| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|30000| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|m7gd.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|233962Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `m7gd.metal` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|m| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|30000| @@ -12243,6 +13004,7 @@ below are the resources available with some assumptions and after the instance o |memory|233962Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## m7i Family ### `m7i.large` #### Labels @@ -12251,6 +13013,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12276,6 +13039,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12301,6 +13065,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12326,6 +13091,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12351,6 +13117,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12376,6 +13143,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12401,6 +13169,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12426,6 +13195,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12451,6 +13221,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12476,6 +13247,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12502,6 +13274,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -12529,6 +13302,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -12554,6 +13328,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -12579,6 +13354,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -12604,6 +13380,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -12629,6 +13406,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -12655,6 +13433,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p2| |karpenter.k8s.aws/instance-generation|2| @@ -12683,6 +13462,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|5000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p2| |karpenter.k8s.aws/instance-generation|2| @@ -12712,6 +13492,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p2| |karpenter.k8s.aws/instance-generation|2| @@ -12742,6 +13523,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p3| |karpenter.k8s.aws/instance-generation|3| @@ -12770,6 +13552,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p3| |karpenter.k8s.aws/instance-generation|3| @@ -12799,6 +13582,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p3| |karpenter.k8s.aws/instance-generation|3| @@ -12829,6 +13613,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|p3dn| |karpenter.k8s.aws/instance-generation|3| @@ -12862,6 +13647,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|p4d| |karpenter.k8s.aws/instance-generation|4| @@ -12895,6 +13681,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|p5| |karpenter.k8s.aws/instance-generation|5| @@ -13045,6 +13832,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|425| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -13069,6 +13857,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|850| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -13093,6 +13882,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1700| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -13117,6 +13907,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -13141,6 +13932,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -13165,6 +13957,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -13190,6 +13983,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13215,6 +14009,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13240,6 +14035,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13265,6 +14061,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13290,6 +14087,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13315,6 +14113,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13340,6 +14139,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13365,6 +14165,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13390,6 +14191,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -13416,6 +14218,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13441,6 +14244,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13466,6 +14270,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13491,6 +14296,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13516,6 +14322,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13541,6 +14348,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13566,6 +14374,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13591,6 +14400,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|13570| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -13617,6 +14427,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13643,6 +14454,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13669,6 +14481,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13695,6 +14508,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2880| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13721,6 +14535,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13747,6 +14562,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|6780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13773,6 +14589,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13799,6 +14616,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|13570| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -13826,6 +14644,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13851,6 +14670,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13876,6 +14696,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13901,6 +14722,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13926,6 +14748,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13951,6 +14774,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13976,6 +14800,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -14001,6 +14826,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -14026,6 +14852,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -14052,6 +14879,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14078,6 +14906,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14104,6 +14933,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14130,6 +14960,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14156,6 +14987,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14182,6 +15014,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14208,6 +15041,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14234,6 +15068,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14260,6 +15095,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -14287,6 +15123,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14313,6 +15150,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14339,6 +15177,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14365,6 +15204,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14391,6 +15231,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14417,6 +15258,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14443,6 +15285,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14469,6 +15312,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14496,6 +15340,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -14524,6 +15369,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14549,6 +15395,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14574,6 +15421,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14599,6 +15447,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14624,6 +15473,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|6800| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14649,6 +15499,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14674,6 +15525,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|13600| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14699,6 +15551,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14725,6 +15578,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -14752,6 +15606,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14777,6 +15632,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14802,6 +15658,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14827,6 +15684,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14852,6 +15710,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14877,6 +15736,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14902,6 +15762,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14927,6 +15788,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14952,6 +15814,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14977,6 +15840,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -15003,6 +15867,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -15030,6 +15895,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15055,6 +15921,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15080,6 +15947,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15105,6 +15973,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15130,6 +15999,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15155,6 +16025,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15180,6 +16051,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15205,6 +16077,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15230,6 +16103,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -15256,6 +16130,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15282,6 +16157,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15308,6 +16184,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15334,6 +16211,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15360,6 +16238,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15386,6 +16265,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15412,6 +16292,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15438,6 +16319,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15464,6 +16346,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -15491,6 +16374,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15516,6 +16400,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15541,6 +16426,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15566,6 +16452,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15591,6 +16478,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15616,6 +16504,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15641,6 +16530,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15666,6 +16556,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15691,6 +16582,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15717,6 +16609,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -15744,6 +16637,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15770,6 +16664,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15796,6 +16691,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15822,6 +16718,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15848,6 +16745,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15874,6 +16772,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15900,6 +16799,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15926,6 +16826,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15952,6 +16853,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -15979,6 +16881,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -16007,6 +16910,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16033,6 +16937,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16059,6 +16964,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16085,6 +16991,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16111,6 +17018,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16137,6 +17045,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|37500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16163,6 +17072,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|50000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16189,6 +17099,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|75000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16215,6 +17126,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16231,10 +17143,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|965782Mi| - |pods|345| + |memory|965243Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ### `r6idn.metal` #### Labels | Label | Value | @@ -16242,6 +17154,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -16258,10 +17171,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|965782Mi| - |pods|345| + |memory|965243Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ## r6in Family ### `r6in.large` #### Labels @@ -16270,6 +17183,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16295,6 +17209,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16320,6 +17235,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16345,6 +17261,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16370,6 +17287,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|25000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16395,6 +17313,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|37500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16420,6 +17339,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|50000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16445,6 +17365,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|75000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16470,6 +17391,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16485,10 +17407,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|965782Mi| - |pods|345| + |memory|965243Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ### `r6in.metal` #### Labels | Label | Value | @@ -16496,6 +17418,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -16511,10 +17434,10 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|965782Mi| - |pods|345| + |memory|965243Mi| + |pods|394| |vpc.amazonaws.com/efa|2| - |vpc.amazonaws.com/pod-eni|108| + |vpc.amazonaws.com/pod-eni|106| ## r7a Family ### `r7a.medium` #### Labels @@ -16523,6 +17446,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16548,6 +17472,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16573,6 +17498,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16598,6 +17524,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16623,6 +17550,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16648,6 +17576,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16673,6 +17602,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16698,6 +17628,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16723,6 +17654,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16748,6 +17680,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16773,6 +17706,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16799,6 +17733,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -16826,6 +17761,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -16851,6 +17787,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -16876,6 +17813,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -16901,6 +17839,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -16926,6 +17865,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -16951,6 +17891,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -16976,6 +17917,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -17001,6 +17943,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -17027,6 +17970,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -17054,6 +17998,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17080,6 +18025,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17106,6 +18052,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17132,6 +18079,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17158,6 +18106,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17184,6 +18133,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17210,6 +18160,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17236,6 +18187,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17263,6 +18215,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -17282,6 +18235,7 @@ below are the resources available with some assumptions and after the instance o |memory|476445Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## r7i Family ### `r7i.large` #### Labels @@ -17290,6 +18244,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17315,6 +18270,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17340,6 +18296,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17365,6 +18322,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17390,6 +18348,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17415,6 +18374,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17440,6 +18400,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17465,6 +18426,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17490,6 +18452,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17515,6 +18478,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17541,6 +18505,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| @@ -17568,6 +18533,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17593,6 +18559,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17618,6 +18585,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17643,6 +18611,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17668,6 +18637,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17693,6 +18663,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17718,6 +18689,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17743,6 +18715,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17768,6 +18741,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17794,6 +18768,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -17813,6 +18788,311 @@ below are the resources available with some assumptions and after the instance o |pods|737| |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| +## r8g Family +### `r8g.medium` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r8g| + |karpenter.k8s.aws/instance-generation|8| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|8192| + |karpenter.k8s.aws/instance-network-bandwidth|520| + |karpenter.k8s.aws/instance-size|medium| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r8g.medium| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|940m| + |ephemeral-storage|17Gi| + |memory|7075Mi| + |pods|8| +### `r8g.large` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r8g| + |karpenter.k8s.aws/instance-generation|8| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|937| + |karpenter.k8s.aws/instance-size|large| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r8g.large| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|1930m| + |ephemeral-storage|17Gi| + |memory|14422Mi| + |pods|29| +### `r8g.xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r8g| + |karpenter.k8s.aws/instance-generation|8| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|1876| + |karpenter.k8s.aws/instance-size|xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r8g.xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|3920m| + |ephemeral-storage|17Gi| + |memory|29258Mi| + |pods|58| +### `r8g.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r8g| + |karpenter.k8s.aws/instance-generation|8| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|3750| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r8g.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|59568Mi| + |pods|58| +### `r8g.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r8g| + |karpenter.k8s.aws/instance-generation|8| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|7500| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r8g.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|118253Mi| + |pods|234| +### `r8g.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|10000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r8g| + |karpenter.k8s.aws/instance-generation|8| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|15000| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r8g.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|239495Mi| + |pods|234| +### `r8g.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|15000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r8g| + |karpenter.k8s.aws/instance-generation|8| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|22500| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r8g.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|360736Mi| + |pods|234| +### `r8g.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r8g| + |karpenter.k8s.aws/instance-generation|8| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|30000| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r8g.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|476445Mi| + |pods|737| +### `r8g.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r8g| + |karpenter.k8s.aws/instance-generation|8| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|40000| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r8g.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|718928Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| +### `r8g.metal-24xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|30000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r8g| + |karpenter.k8s.aws/instance-generation|8| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|40000| + |karpenter.k8s.aws/instance-size|metal-24xl| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r8g.metal-24xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|718928Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| +### `r8g.48xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r8g| + |karpenter.k8s.aws/instance-generation|8| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|48xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r8g.48xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|1446378Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| +### `r8g.metal-48xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r8g| + |karpenter.k8s.aws/instance-generation|8| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|metal-48xl| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r8g.metal-48xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|1446378Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| ## t1 Family ### `t1.micro` #### Labels @@ -18007,6 +19287,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18031,6 +19312,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18055,6 +19337,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18079,6 +19362,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18103,6 +19387,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18127,6 +19412,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18151,6 +19437,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -18176,6 +19463,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18200,6 +19488,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18224,6 +19513,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18248,6 +19538,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18272,6 +19563,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18296,6 +19588,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18320,6 +19613,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|amd| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -18345,6 +19639,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18369,6 +19664,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18393,6 +19689,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18417,6 +19714,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2085| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18441,6 +19739,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18465,6 +19764,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18489,6 +19789,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|2780| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -18517,6 +19818,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|trn| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|trn1| |karpenter.k8s.aws/instance-generation|1| @@ -18547,6 +19849,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|trn| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|trn1| |karpenter.k8s.aws/instance-generation|1| @@ -18579,6 +19882,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|trn| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|trn1n| |karpenter.k8s.aws/instance-generation|1| @@ -18608,6 +19912,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-12tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18633,6 +19938,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-18tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18658,6 +19964,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-24tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18683,6 +19990,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|224| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-3tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18709,6 +20017,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|224| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-6tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18733,6 +20042,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-6tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18758,6 +20068,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|38000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-9tb1| |karpenter.k8s.aws/instance-generation|1| @@ -18775,6 +20086,114 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|8720933Mi| |pods|737| +## u7i-12tb Family +### `u7i-12tb.224xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|u| + |karpenter.k8s.aws/instance-cpu|896| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|u7i-12tb| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|12582912| + |karpenter.k8s.aws/instance-network-bandwidth|100000| + |karpenter.k8s.aws/instance-size|224xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|u7i-12tb.224xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|893690m| + |ephemeral-storage|17Gi| + |memory|11630731Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| +## u7in-16tb Family +### `u7in-16tb.224xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|u| + |karpenter.k8s.aws/instance-cpu|896| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|u7in-16tb| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|16777216| + |karpenter.k8s.aws/instance-network-bandwidth|200000| + |karpenter.k8s.aws/instance-size|224xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|u7in-16tb.224xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|893690m| + |ephemeral-storage|17Gi| + |memory|15514235Mi| + |pods|394| + |vpc.amazonaws.com/efa|2| +## u7in-24tb Family +### `u7in-24tb.224xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|u| + |karpenter.k8s.aws/instance-cpu|896| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|u7in-24tb| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|25165824| + |karpenter.k8s.aws/instance-network-bandwidth|200000| + |karpenter.k8s.aws/instance-size|224xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|u7in-24tb.224xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|893690m| + |ephemeral-storage|17Gi| + |memory|23273698Mi| + |pods|394| + |vpc.amazonaws.com/efa|2| +## u7in-32tb Family +### `u7in-32tb.224xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|u| + |karpenter.k8s.aws/instance-cpu|896| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|100000| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|u7in-32tb| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|33554432| + |karpenter.k8s.aws/instance-network-bandwidth|200000| + |karpenter.k8s.aws/instance-size|224xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|u7in-32tb.224xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|893690m| + |ephemeral-storage|17Gi| + |memory|31033160Mi| + |pods|394| + |vpc.amazonaws.com/efa|2| ## vt1 Family ### `vt1.3xlarge` #### Labels @@ -18783,6 +20202,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|vt| |karpenter.k8s.aws/instance-cpu|12| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|vt1| |karpenter.k8s.aws/instance-generation|1| @@ -18808,6 +20228,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|vt| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|vt1| |karpenter.k8s.aws/instance-generation|1| @@ -18833,6 +20254,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|vt| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|vt1| |karpenter.k8s.aws/instance-generation|1| @@ -18860,6 +20282,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1| |karpenter.k8s.aws/instance-generation|1| @@ -18884,6 +20307,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1| |karpenter.k8s.aws/instance-generation|1| @@ -18909,6 +20333,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -18933,6 +20358,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -18957,6 +20383,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|1750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -18981,6 +20408,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -19005,6 +20433,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|7000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -19029,6 +20458,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|14000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -19054,6 +20484,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19080,6 +20511,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19106,6 +20538,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19132,6 +20565,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19158,6 +20592,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19184,6 +20619,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19210,6 +20646,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|14250| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19236,6 +20673,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19262,6 +20700,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -19289,6 +20728,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2idn| |karpenter.k8s.aws/instance-generation|2| @@ -19315,6 +20755,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2idn| |karpenter.k8s.aws/instance-generation|2| @@ -19341,6 +20782,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2idn| |karpenter.k8s.aws/instance-generation|2| @@ -19368,6 +20810,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2idn| |karpenter.k8s.aws/instance-generation|2| @@ -19396,6 +20839,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19422,6 +20866,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19448,6 +20893,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19474,6 +20920,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|20000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19500,6 +20947,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|40000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19526,6 +20974,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|60000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19552,6 +21001,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19579,6 +21029,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|80000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -19607,6 +21058,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -19632,6 +21084,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -19657,6 +21110,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -19682,6 +21136,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|12000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -19707,6 +21162,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -19733,6 +21189,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -19760,6 +21217,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -19786,6 +21244,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -19812,6 +21271,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|3170| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -19838,6 +21298,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|12| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|4750| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -19864,6 +21325,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|24| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|9500| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -19890,6 +21352,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -19916,6 +21379,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-cpu-manufacturer|intel| + |karpenter.k8s.aws/instance-ebs-bandwidth|19000| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| diff --git a/website/content/en/preview/reference/metrics.md b/website/content/en/preview/reference/metrics.md index c717023a2cad..674dc859179c 100644 --- a/website/content/en/preview/reference/metrics.md +++ b/website/content/en/preview/reference/metrics.md @@ -7,196 +7,304 @@ description: > Inspect Karpenter Metrics --- -Karpenter makes several metrics available in Prometheus format to allow monitoring cluster provisioning status. These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8000/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings) +Karpenter makes several metrics available in Prometheus format to allow monitoring cluster provisioning status. These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8080/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings) ### `karpenter_build_info` A metric with a constant '1' value labeled by version from which karpenter was built. +- Stability Level: STABLE -## Nodepool Metrics +## Nodeclaims Metrics + +### `karpenter_nodeclaims_termination_duration_seconds` +Duration of NodeClaim termination in seconds. +- Stability Level: BETA -### `karpenter_nodepool_usage` -The nodepool usage is the amount of resources that have been provisioned by a particular nodepool. Labeled by nodepool name and resource type. +### `karpenter_nodeclaims_terminated_total` +Number of nodeclaims terminated in total by Karpenter. Labeled by the owning nodepool. +- Stability Level: STABLE -### `karpenter_nodepool_limit` -The nodepool limits are the limits specified on the nodepool that restrict the quantity of resources provisioned. Labeled by nodepool name and resource type. +### `karpenter_nodeclaims_instance_termination_duration_seconds` +Duration of CloudProvider Instance termination in seconds. +- Stability Level: BETA + +### `karpenter_nodeclaims_disrupted_total` +Number of nodeclaims disrupted in total by Karpenter. Labeled by reason the nodeclaim was disrupted and the owning nodepool. +- Stability Level: ALPHA + +### `karpenter_nodeclaims_created_total` +Number of nodeclaims created in total by Karpenter. Labeled by reason the nodeclaim was created and the owning nodepool. +- Stability Level: STABLE ## Nodes Metrics ### `karpenter_nodes_total_pod_requests` -Node total pod requests are the resources requested by non-DaemonSet pods bound to nodes. +Node total pod requests are the resources requested by pods bound to nodes, including the DaemonSet pods. +- Stability Level: BETA ### `karpenter_nodes_total_pod_limits` -Node total pod limits are the resources specified by non-DaemonSet pod limits. +Node total pod limits are the resources specified by pod limits, including the DaemonSet pods. +- Stability Level: BETA ### `karpenter_nodes_total_daemon_requests` Node total daemon requests are the resource requested by DaemonSet pods bound to nodes. +- Stability Level: BETA ### `karpenter_nodes_total_daemon_limits` Node total daemon limits are the resources specified by DaemonSet pod limits. +- Stability Level: BETA -### `karpenter_nodes_termination_time_seconds` +### `karpenter_nodes_termination_duration_seconds` The time taken between a node's deletion request and the removal of its finalizer +- Stability Level: BETA -### `karpenter_nodes_terminated` +### `karpenter_nodes_terminated_total` Number of nodes terminated in total by Karpenter. Labeled by owning nodepool. +- Stability Level: STABLE ### `karpenter_nodes_system_overhead` Node system daemon overhead are the resources reserved for system overhead, the difference between the node's capacity and allocatable values are reported by the status. +- Stability Level: BETA -### `karpenter_nodes_leases_deleted` +### `karpenter_nodes_leases_deleted_total` Number of deleted leaked leases. +- Stability Level: ALPHA -### `karpenter_nodes_eviction_queue_depth` -The number of pods currently waiting for a successful eviction in the eviction queue. - -### `karpenter_nodes_created` +### `karpenter_nodes_created_total` Number of nodes created in total by Karpenter. Labeled by owning nodepool. +- Stability Level: STABLE ### `karpenter_nodes_allocatable` Node allocatable are the resources allocatable by nodes. +- Stability Level: BETA ## Pods Metrics ### `karpenter_pods_state` Pod state is the current state of pods. This metric can be used several ways as it is labeled by the pod name, namespace, owner, node, nodepool name, zone, architecture, capacity type, instance type and pod phase. +- Stability Level: BETA -### `karpenter_pods_startup_time_seconds` +### `karpenter_pods_startup_duration_seconds` The time from pod creation until the pod is running. +- Stability Level: STABLE -## Provisioner Metrics +## Voluntary Disruption Metrics -### `karpenter_provisioner_scheduling_simulation_duration_seconds` -Duration of scheduling simulations used for deprovisioning and provisioning in seconds. +### `karpenter_voluntary_disruption_queue_failures_total` +The number of times that an enqueued disruption decision failed. Labeled by disruption method. +- Stability Level: BETA -### `karpenter_provisioner_scheduling_queue_depth` -The number of pods currently waiting to be scheduled. +### `karpenter_voluntary_disruption_eligible_nodes` +Number of nodes eligible for disruption by Karpenter. Labeled by disruption reason. +- Stability Level: BETA -### `karpenter_provisioner_scheduling_duration_seconds` -Duration of scheduling process in seconds. +### `karpenter_voluntary_disruption_decisions_total` +Number of disruption decisions performed. Labeled by disruption decision, reason, and consolidation type. +- Stability Level: STABLE -## Nodeclaims Metrics +### `karpenter_voluntary_disruption_decision_evaluation_duration_seconds` +Duration of the disruption decision evaluation process in seconds. Labeled by method and consolidation type. +- Stability Level: BETA -### `karpenter_nodeclaims_terminated` -Number of nodeclaims terminated in total by Karpenter. Labeled by reason the nodeclaim was terminated and the owning nodepool. +### `karpenter_voluntary_disruption_consolidation_timeouts_total` +Number of times the Consolidation algorithm has reached a timeout. Labeled by consolidation type. +- Stability Level: BETA -### `karpenter_nodeclaims_registered` -Number of nodeclaims registered in total by Karpenter. Labeled by the owning nodepool. +## Scheduler Metrics -### `karpenter_nodeclaims_launched` -Number of nodeclaims launched in total by Karpenter. Labeled by the owning nodepool. +### `karpenter_scheduler_scheduling_duration_seconds` +Duration of scheduling simulations used for deprovisioning and provisioning in seconds. +- Stability Level: STABLE -### `karpenter_nodeclaims_initialized` -Number of nodeclaims initialized in total by Karpenter. Labeled by the owning nodepool. +### `karpenter_scheduler_queue_depth` +The number of pods currently waiting to be scheduled. +- Stability Level: BETA -### `karpenter_nodeclaims_drifted` -Number of nodeclaims drifted reasons in total by Karpenter. Labeled by drift type of the nodeclaim and the owning nodepool. +## Nodepools Metrics -### `karpenter_nodeclaims_disrupted` -Number of nodeclaims disrupted in total by Karpenter. Labeled by disruption type of the nodeclaim and the owning nodepool. +### `karpenter_nodepools_usage` +The amount of resources that have been provisioned for a nodepool. Labeled by nodepool name and resource type. +- Stability Level: ALPHA -### `karpenter_nodeclaims_created` -Number of nodeclaims created in total by Karpenter. Labeled by reason the nodeclaim was created and the owning nodepool. +### `karpenter_nodepools_limit` +Limits specified on the nodepool that restrict the quantity of resources provisioned. Labeled by nodepool name and resource type. +- Stability Level: ALPHA + +### `karpenter_nodepools_allowed_disruptions` +The number of nodes for a given NodePool that can be concurrently disrupting at a point in time. Labeled by NodePool. Note that allowed disruptions can change very rapidly, as new nodes may be created and others may be deleted at any point. +- Stability Level: ALPHA ## Interruption Metrics -### `karpenter_interruption_received_messages` +### `karpenter_interruption_received_messages_total` Count of messages received from the SQS queue. Broken down by message type and whether the message was actionable. +- Stability Level: STABLE -### `karpenter_interruption_message_latency_time_seconds` -Length of time between message creation in queue and an action taken on the message by the controller. +### `karpenter_interruption_message_queue_duration_seconds` +Amount of time an interruption message is on the queue before it is processed by karpenter. +- Stability Level: STABLE -### `karpenter_interruption_deleted_messages` +### `karpenter_interruption_deleted_messages_total` Count of messages deleted from the SQS queue. - -### `karpenter_interruption_actions_performed` -Number of notification actions performed. Labeled by action - -## Disruption Metrics - -### `karpenter_disruption_replacement_nodeclaim_initialized_seconds` -Amount of time required for a replacement nodeclaim to become initialized. - -### `karpenter_disruption_replacement_nodeclaim_failures_total` -The number of times that Karpenter failed to launch a replacement node for disruption. Labeled by disruption method. - -### `karpenter_disruption_queue_depth` -The number of commands currently being waited on in the disruption orchestration queue. - -### `karpenter_disruption_pods_disrupted_total` -Total number of reschedulable pods disrupted on nodes. Labeled by NodePool, disruption action, method, and consolidation type. - -### `karpenter_disruption_nodes_disrupted_total` -Total number of nodes disrupted. Labeled by NodePool, disruption action, method, and consolidation type. - -### `karpenter_disruption_evaluation_duration_seconds` -Duration of the disruption evaluation process in seconds. Labeled by method and consolidation type. - -### `karpenter_disruption_eligible_nodes` -Number of nodes eligible for disruption by Karpenter. Labeled by disruption method and consolidation type. - -### `karpenter_disruption_consolidation_timeouts_total` -Number of times the Consolidation algorithm has reached a timeout. Labeled by consolidation type. - -### `karpenter_disruption_budgets_allowed_disruptions` -The number of nodes for a given NodePool that can be disrupted at a point in time. Labeled by NodePool. Note that allowed disruptions can change very rapidly, as new nodes may be created and others may be deleted at any point. - -### `karpenter_disruption_actions_performed_total` -Number of disruption actions performed. Labeled by disruption action, method, and consolidation type. - -## Consistency Metrics - -### `karpenter_consistency_errors` -Number of consistency checks that have failed. +- Stability Level: STABLE ## Cluster State Metrics ### `karpenter_cluster_state_synced` Returns 1 if cluster state is synced and 0 otherwise. Synced checks that nodeclaims and nodes that are stored in the APIServer have the same representation as Karpenter's cluster state +- Stability Level: STABLE ### `karpenter_cluster_state_node_count` Current count of nodes in cluster state +- Stability Level: STABLE ## Cloudprovider Metrics ### `karpenter_cloudprovider_instance_type_offering_price_estimate` Instance type offering estimated hourly price used when making informed decisions on node cost calculation, based on instance type, capacity type, and zone. +- Stability Level: BETA ### `karpenter_cloudprovider_instance_type_offering_available` Instance type offering availability, based on instance type, capacity type, and zone +- Stability Level: BETA ### `karpenter_cloudprovider_instance_type_memory_bytes` Memory, in bytes, for a given instance type. +- Stability Level: BETA ### `karpenter_cloudprovider_instance_type_cpu_cores` VCPUs cores for a given instance type. +- Stability Level: BETA ### `karpenter_cloudprovider_errors_total` Total number of errors returned from CloudProvider calls. +- Stability Level: BETA ### `karpenter_cloudprovider_duration_seconds` Duration of cloud provider method calls. Labeled by the controller, method name and provider. +- Stability Level: BETA ## Cloudprovider Batcher Metrics ### `karpenter_cloudprovider_batcher_batch_time_seconds` Duration of the batching window per batcher +- Stability Level: BETA ### `karpenter_cloudprovider_batcher_batch_size` Size of the request batch per batcher +- Stability Level: BETA ## Controller Runtime Metrics +### `controller_runtime_terminal_reconcile_errors_total` +Total number of terminal reconciliation errors per controller +- Stability Level: STABLE + ### `controller_runtime_reconcile_total` Total number of reconciliations per controller +- Stability Level: STABLE ### `controller_runtime_reconcile_time_seconds` Length of time per reconciliation per controller +- Stability Level: STABLE ### `controller_runtime_reconcile_errors_total` Total number of reconciliation errors per controller +- Stability Level: STABLE ### `controller_runtime_max_concurrent_reconciles` Maximum number of concurrent reconciles per controller +- Stability Level: STABLE ### `controller_runtime_active_workers` Number of currently used workers per controller +- Stability Level: STABLE + +## Workqueue Metrics + +### `workqueue_work_duration_seconds` +How long in seconds processing an item from workqueue takes. +- Stability Level: STABLE + +### `workqueue_unfinished_work_seconds` +How many seconds of work has been done that is in progress and hasn't been observed by work_duration. Large values indicate stuck threads. One can deduce the number of stuck threads by observing the rate at which this increases. +- Stability Level: STABLE + +### `workqueue_retries_total` +Total number of retries handled by workqueue +- Stability Level: STABLE + +### `workqueue_queue_duration_seconds` +How long in seconds an item stays in workqueue before being requested +- Stability Level: STABLE + +### `workqueue_longest_running_processor_seconds` +How many seconds has the longest running processor for workqueue been running. +- Stability Level: STABLE + +### `workqueue_depth` +Current depth of workqueue +- Stability Level: STABLE + +### `workqueue_adds_total` +Total number of adds handled by workqueue +- Stability Level: STABLE + +## Status Condition Metrics + +### `operator_status_condition_transitions_total` +The count of transitions of a given object, type and status. +- Stability Level: BETA + +### `operator_status_condition_transition_seconds` +The amount of time a condition was in a given state before transitioning. e.g. Alarm := P99(Updated=False) > 5 minutes +- Stability Level: BETA + +### `operator_status_condition_current_status_seconds` +The current amount of time in seconds that a status condition has been in a specific state. Alarm := P99(Updated=Unknown) > 5 minutes +- Stability Level: BETA + +### `operator_status_condition_count` +The number of an condition for a given object, type and status. e.g. Alarm := Available=False > 0 +- Stability Level: BETA + +## Client Go Metrics + +### `client_go_request_total` +Number of HTTP requests, partitioned by status code and method. +- Stability Level: STABLE + +### `client_go_request_duration_seconds` +Request latency in seconds. Broken down by verb, group, version, kind, and subresource. +- Stability Level: STABLE + +## AWS SDK Go Metrics + +### `aws_sdk_go_request_total` +The total number of AWS SDK Go requests +- Stability Level: STABLE + +### `aws_sdk_go_request_retry_count` +The total number of AWS SDK Go retry attempts per request +- Stability Level: STABLE + +### `aws_sdk_go_request_duration_seconds` +Latency of AWS SDK Go requests +- Stability Level: STABLE + +### `aws_sdk_go_request_attempt_total` +The total number of AWS SDK Go request attempts +- Stability Level: STABLE + +### `aws_sdk_go_request_attempt_duration_seconds` +Latency of AWS SDK Go request attempts +- Stability Level: STABLE + +## Leader Election Metrics + +### `leader_election_slowpath_total` +Total number of slow path exercised in renewing leader leases. 'name' is the string used to identify the lease. Please make sure to group by name. +- Stability Level: STABLE + +### `leader_election_master_status` +Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease. Please make sure to group by name. +- Stability Level: STABLE diff --git a/website/content/en/preview/reference/settings.md b/website/content/en/preview/reference/settings.md index 753e9ef7b707..3374e416745a 100644 --- a/website/content/en/preview/reference/settings.md +++ b/website/content/en/preview/reference/settings.md @@ -12,26 +12,26 @@ Karpenter surfaces environment variables and CLI parameters to allow you to conf | Environment Variable | CLI Flag | Description | |--|--|--| -| ASSUME_ROLE_ARN | \-\-assume-role-arn | Role to assume for calling AWS services.| -| ASSUME_ROLE_DURATION | \-\-assume-role-duration | Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set. (default = 15m0s)| | BATCH_IDLE_DURATION | \-\-batch-idle-duration | The maximum amount of time with no new pending pods that if exceeded ends the current batching window. If pods arrive faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods will be batched separately. (default = 1s)| | BATCH_MAX_DURATION | \-\-batch-max-duration | The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one time which usually results in fewer but larger nodes. (default = 10s)| | CLUSTER_CA_BUNDLE | \-\-cluster-ca-bundle | Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.| | CLUSTER_ENDPOINT | \-\-cluster-endpoint | The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.| | CLUSTER_NAME | \-\-cluster-name | [REQUIRED] The kubernetes cluster name for resource discovery.| -| DISABLE_WEBHOOK | \-\-disable-webhook | Disable the admission and validation webhooks| +| DISABLE_LEADER_ELECTION | \-\-disable-leader-election | Disable the leader election client before executing the main loop. Disable when running replicated components for high availability is not desired.| +| DISABLE_WEBHOOK | \-\-disable-webhook | Disable the conversion webhooks| | ENABLE_PROFILING | \-\-enable-profiling | Enable the profiling on the metric endpoint| -| FEATURE_GATES | \-\-feature-gates | Optional features can be enabled / disabled using feature gates. Current options are: Drift,SpotToSpotConsolidation (default = Drift=true,SpotToSpotConsolidation=false)| +| FEATURE_GATES | \-\-feature-gates | Optional features can be enabled / disabled using feature gates. Current options are: SpotToSpotConsolidation (default = SpotToSpotConsolidation=false)| | HEALTH_PROBE_PORT | \-\-health-probe-port | The port the health probe endpoint binds to for reporting controller health (default = 8081)| -| INTERRUPTION_QUEUE | \-\-interruption-queue | Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.| +| INTERRUPTION_QUEUE | \-\-interruption-queue | Interruption queue is the name of the SQS queue used for processing interruption events from EC2. Interruption handling is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.| | ISOLATED_VPC | \-\-isolated-vpc | If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS on-demand pricing endpoint.| | KARPENTER_SERVICE | \-\-karpenter-service | The Karpenter Service name for the dynamic webhook certificate| | KUBE_CLIENT_BURST | \-\-kube-client-burst | The maximum allowed burst of queries to the kube-apiserver (default = 300)| | KUBE_CLIENT_QPS | \-\-kube-client-qps | The smoothed rate of qps to kube-apiserver (default = 200)| -| LEADER_ELECT | \-\-leader-elect | Start leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.| +| LOG_ERROR_OUTPUT_PATHS | \-\-log-error-output-paths | Optional comma separated paths for logging error output (default = stderr)| | LOG_LEVEL | \-\-log-level | Log verbosity level. Can be one of 'debug', 'info', or 'error' (default = info)| +| LOG_OUTPUT_PATHS | \-\-log-output-paths | Optional comma separated paths for directing log output (default = stdout)| | MEMORY_LIMIT | \-\-memory-limit | Memory limit on the container running the controller. The GC soft memory limit is set to 90% of this value. (default = -1)| -| METRICS_PORT | \-\-metrics-port | The port the metric endpoint binds to for operating metrics about the controller itself (default = 8000)| +| METRICS_PORT | \-\-metrics-port | The port the metric endpoint binds to for operating metrics about the controller itself (default = 8080)| | RESERVED_ENIS | \-\-reserved-enis | Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html. (default = 0)| | VM_MEMORY_OVERHEAD_PERCENT | \-\-vm-memory-overhead-percent | The VM memory overhead as a percent that will be subtracted from the total memory for all instance types. (default = 0.075)| | WEBHOOK_METRICS_PORT | \-\-webhook-metrics-port | The port the webhook metric endpoing binds to for operating metrics about the webhook (default = 8001)| diff --git a/website/content/en/preview/tasks/managing-amis.md b/website/content/en/preview/tasks/managing-amis.md index 1ef31f141e08..47d2b3bab9b1 100644 --- a/website/content/en/preview/tasks/managing-amis.md +++ b/website/content/en/preview/tasks/managing-amis.md @@ -58,7 +58,7 @@ For example, you could have: * **Test clusters**: On lower environment clusters, you can run the latest AMIs for your workloads in a safe environment. The `EC2NodeClass` for these clusters could be set with a chosen `amiFamily`, but no `amiSelectorTerms` set. For example, the `NodePool` and `EC2NodeClass` could begin with the following: ```yaml - apiVersion: karpenter.sh/v1beta1 + apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -66,11 +66,11 @@ For example, you could have: template: spec: nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass name: default --- - apiVersion: karpenter.k8s.aws/v1beta1 + apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: default @@ -120,9 +120,11 @@ You can set Disruption Budgets in a `NodePool` spec. Here is an example: ```yaml +template: + spec: + expireAfter: 1440h disruption: consolidationPolicy: WhenEmpty - expireAfter: 1440h budgets: - nodes: 15% - nodes: "3" @@ -132,7 +134,7 @@ disruption: ``` The `disruption` settings define a few fields that indicate the state of a node that should be disrupted. -The `consolidationPolicy` field indicates that a node should be disrupted if the node is either underutilized (`WhenUnderutilized`) or not running any pods (`WhenEmpty`). +The `consolidationPolicy` field indicates that a node should be disrupted if the node is either empty or underutilized (`WhenEmptyOrUnderutilized`) or not running any pods (`WhenEmpty`). With `expireAfter` set to `1440` hours, the node expires after 60 days. Extending those values causes longer times without disruption. diff --git a/website/content/en/preview/troubleshooting.md b/website/content/en/preview/troubleshooting.md index 6dc784007b75..2627508f2954 100644 --- a/website/content/en/preview/troubleshooting.md +++ b/website/content/en/preview/troubleshooting.md @@ -73,7 +73,7 @@ Node role names for Karpenter are created in the form `KarpenterNodeRole-${Clust If a long cluster name causes the Karpenter node role name to exceed 64 characters, creating that object will fail. Keep in mind that `KarpenterNodeRole-` is just a recommendation from the getting started guide. -Instead using of the eksctl role, you can shorten the name to anything you like, as long as it has the right permissions. +Instead of using the eksctl role, you can shorten the name to anything you like, as long as it has the right permissions. ### Unknown field in Provisioner spec @@ -123,8 +123,9 @@ kubectl label crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodecl - In the case of `annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "karpenter"` run: ```shell +KARPENTER_NAMESPACE=kube-system kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite -kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace="${KARPENTER_NAMESPACE}" --overwrite ``` ## Uninstallation @@ -304,7 +305,7 @@ The following is a list of known CSI drivers which support a startupTaint to eli These taints should be configured via `startupTaints` on your `NodePool`. For example, to enable this for EBS, add the following to your `NodePool`: ```yaml -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool spec: template: @@ -330,6 +331,11 @@ By default, the number of pods on a node is limited by both the number of networ If the max-pods (configured through your Provisioner [`kubeletConfiguration`]({{}})) is greater than the number of supported IPs for a given instance type, the CNI will fail to assign an IP to the pod and your pod will be left in a `ContainerCreating` state. +If you've enabled [Security Groups per Pod](https://aws.github.io/aws-eks-best-practices/networking/sgpp/), one of the instance's ENIs is reserved as the trunk interface and uses branch interfaces off of that trunk interface to assign different security groups. +If you do not have any `SecurityGroupPolicies` configured for your pods, they will be unable to utilize branch interfaces attached to the trunk interface, and IPs will only be available from the non-trunk ENIs. +This effectively reduces the max-pods value by the number of IPs that would have been available from the trunk ENI. +Note that Karpenter is not aware if [Security Groups per Pod](https://aws.github.io/aws-eks-best-practices/networking/sgpp/) is enabled, and will continue to compute max-pods assuming all ENIs on the instance can be utilized. + ##### Solutions To avoid this discrepancy between `maxPods` and the supported pod density of the EC2 instance based on ENIs and allocatable IPs, you can perform one of the following actions on your cluster: @@ -647,7 +653,7 @@ To correct the problem if it occurs, you can use the approach that AWS EBS uses, "AWS": "arn:aws:iam::${AWS_ACCOUNT_ID}:root" }, "Action": [ - "kms:Describe", + "kms:Describe*", "kms:Get*", "kms:List*", "kms:RevokeGrant" @@ -663,7 +669,7 @@ This typically occurs when the node has not been considered fully initialized fo ### Log message of `inflight check failed for node, Expected resource "vpc.amazonaws.com/pod-eni" didn't register on the node` is reported -This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. If you've enabled Pod ENI for Karpenter nodes via the `aws.enablePodENI` setting, you will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. +This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. You will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. ### AWS Node Termination Handler (NTH) interactions Karpenter [doesn't currently support draining and terminating on spot rebalance recommendations]({{< ref "concepts/disruption#interruption" >}}). Users who want support for both drain and terminate on spot interruption as well as drain and termination on spot rebalance recommendations may install Node Termination Handler (NTH) on their clusters to support this behavior. diff --git a/website/content/en/preview/upgrading/compatibility.md b/website/content/en/preview/upgrading/compatibility.md index 0f2c8d6cab94..2ccb7d2c9d1e 100644 --- a/website/content/en/preview/upgrading/compatibility.md +++ b/website/content/en/preview/upgrading/compatibility.md @@ -15,9 +15,9 @@ Before you begin upgrading Karpenter, consider Karpenter compatibility issues re [comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) -| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | 1.29 | -|------------|----------|----------|----------|----------|----------|----------|------------| -| karpenter | \>= 0.21 | \>= 0.21 | \>= 0.25 | \>= 0.28 | \>= 0.28 | \>= 0.31 | \>= 0.34.0 | +| KUBERNETES | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | 1.29 | 1.30 | +|------------|----------|----------|----------|----------|----------|----------|--------| +| karpenter | \>= 0.21 | \>= 0.25 | \>= 0.28 | \>= 0.28 | \>= 0.31 | \>= 0.34 | 0.37.0 | [comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) @@ -75,14 +75,13 @@ Karpenter offers three types of releases. This section explains the purpose of e ### Stable Releases -Stable releases are the most reliable releases that are released with weekly cadence. Stable releases are our only recommended versions for production environments. -Sometimes we skip a stable release because we find instability or problems that need to be fixed before having a stable release. -Stable releases are tagged with a semantic version prefixed by a `v`. For example `v0.13.0`. +Stable releases are the only recommended versions for production environments. Stable releases are tagged with a semantic version (e.g. `0.35.0`). Note that stable releases prior to `0.35.0` are prefixed with a `v` (e.g. `v0.34.0`). ### Release Candidates -We consider having release candidates for major and important minor versions. Our release candidates are tagged like `vx.y.z-rc.0`, `vx.y.z-rc.1`. The release candidate will then graduate to `vx.y.z` as a normal stable release. +We consider having release candidates for major and important minor versions. Our release candidates are tagged like `x.y.z-rc.0`, `x.y.z-rc.1`. The release candidate will then graduate to `x.y.z` as a stable release. By adopting this practice we allow our users who are early adopters to test out new releases before they are available to the wider community, thereby providing us with early feedback resulting in more stable releases. +Note that, like the stable releases, release candidates prior to `0.35.0` are prefixed with a `v`. ### Snapshot Releases diff --git a/website/content/en/preview/upgrading/upgrade-guide.md b/website/content/en/preview/upgrading/upgrade-guide.md index b991ad2dd856..8fbc20aba9c3 100644 --- a/website/content/en/preview/upgrading/upgrade-guide.md +++ b/website/content/en/preview/upgrading/upgrade-guide.md @@ -10,6 +10,10 @@ Karpenter is a controller that runs in your cluster, but it is not tied to a spe Use your existing upgrade mechanisms to upgrade your core add-ons in Kubernetes and keep Karpenter up to date on bug fixes and new features. This guide contains information needed to upgrade to the latest release of Karpenter, along with compatibility issues you need to be aware of when upgrading from earlier Karpenter versions. +{{% alert title="Warning" color="warning" %}} +With the release of Karpenter v1.0.0, the Karpenter team will be dropping support for karpenter versions v0.32 and below. We recommend upgrading to the latest version of Karpenter and keeping Karpenter up-to-date for bug fixes and new features. +{{% /alert %}} + ### CRD Upgrades Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: @@ -25,21 +29,65 @@ If you get the error `invalid ownership metadata; label validation error:` while * As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the Helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. -In general, you can reapply the CRDs in the `crds` directory of the Karpenter Helm chart: - -```shell -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.sh_nodepools.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.sh_nodeclaims.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml -``` - +### Upgrading to `1.0.0`+ + +{{% alert title="Warning" color="warning" %}} +Karpenter `1.0.0` introduces v1 APIs, including _significant_ changes to the API and upgrade procedures for the Karpenter controllers. **Do not** upgrade to `1.0.0`+ without referencing the [v1 Migration Upgrade Procedure]({{}}). + +This version adds [conversion webhooks](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/#webhook-conversion) to automatically pull the v1 API version of previously applied v1beta1 NodePools, EC2NodeClasses, and NodeClaims. Karpenter will stop serving the v1beta1 API version at v1.1.0 and will drop the conversion webhooks at that time. Migrate all stored manifests to v1 API versions on Karpenter v1.0+. +{{% /alert %}} + +Below is the full changelog for v1, copied from the [v1 Migration Upgrade Procedure]({{}}). + +* Features: + * AMI Selector Terms has a new Alias field which can only be set by itself in `EC2NodeClass.Spec.AMISelectorTerms` + * Disruption Budgets by Reason was added to `NodePool.Spec.Disruption.Budgets` + * TerminationGracePeriod was added to `NodePool.Spec.Template.Spec`. + * LOG_OUTPUT_PATHS and LOG_ERROR_OUTPUT_PATHS environment variables added +* API Rename: NodePool’s ConsolidationPolicy `WhenUnderutilized` is now renamed to `WhenEmptyOrUnderutilized` +* Behavior Changes: + * Expiration is now forceful and begins draining as soon as it’s expired. Karpenter does not wait for replacement capacity to be available before draining, but will start provisioning a replacement as soon as the node is expired and begins draining. + * Karpenter's generated NodeConfig now takes precedence when generating UserData with the AL2023 `amiFamily`. If you're setting any values managed by Karpenter in your AL2023 UserData, configure these through Karpenter natively (e.g. kubelet configuration fields). + * Karpenter now adds a `karpenter.sh/unregistered:NoExecute` taint to nodes in injected UserData when using alias in AMISelectorTerms or non-Custom AMIFamily. When using `amiFamily: Custom`, users will need to add this taint into their UserData, where Karpenter will automatically remove it when provisioning nodes. +* API Moves: + * ExpireAfter has moved from the `NodePool.Spec.Disruption` block to `NodePool.Spec.Template.Spec`, and is now a drift-able field. + * `Kubelet` was moved to the EC2NodeClass from the NodePool. +* RBAC changes: added `delete pods` | added `get, patch crds` | added `update nodes` | removed `create nodes` +* Breaking API (Manual Migration Needed): + * Ubuntu is dropped as a first class supported AMI Family + * `karpenter.sh/do-not-consolidate` (annotation), `karpenter.sh/do-not-evict` (annotation), and `karpenter.sh/managed-by` (tag) are all removed. `karpenter.sh/managed-by`, which currently stores the cluster name in its value, will be replaced by eks:eks-cluster-name + * The taint used to mark nodes for disruption and termination changed from `karpenter.sh/disruption=disrupting:NoSchedule` to `karpenter.sh/disrupted:NoSchedule`. It is not recommended to tolerate this taint, however, if you were tolerating it in your applications, you'll need to adjust your taints to reflect this. +* Environment Variable Changes: + * Environment Variable Changes + * LOGGING_CONFIG, ASSUME_ROLE_ARN, ASSUME_ROLE_DURATION Dropped + * LEADER_ELECT renamed to DISABLE_LEADER_ELECTION + * `FEATURE_GATES.DRIFT=true` was dropped and promoted to Stable, and cannot be disabled. + * Users currently opting out of drift, disabling the drift feature flag will no longer be able to do so. +* Defaults changed: + * API: Karpenter will drop support for IMDS access from containers by default on new EC2NodeClasses by updating the default of `httpPutResponseHopLimit` from 2 to 1. + * API: ConsolidateAfter is required. Users couldn’t set this before with ConsolidationPolicy: WhenUnderutilized, where this is now required. Users can set it to 0 to have the same behavior as in v1beta1. + * API: All `NodeClassRef` fields are now all required, and apiVersion has been renamed to group + * API: AMISelectorTerms are required. Setting an Alias cannot be done with any other type of term, and must match the AMI Family that's set or be Custom. + * Helm: Deployment spec TopologySpreadConstraint to have required zonal spread over preferred. Users who had one node running their Karpenter deployments need to either: + * Have two nodes in different zones to ensure both Karpenter replicas schedule + * Scale down their Karpenter replicas from 2 to 1 in the helm chart + * Edit and relax the topology spread constraint in their helm chart from DoNotSchedule to ScheduleAnyway + * Helm/Binary: `controller.METRICS_PORT` default changed back to 8080 + ### Upgrading to `0.37.0`+ +{{% alert title="Warning" color="warning" %}} +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to `0.37.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} + +* Karpenter now adds a readiness status condition to the EC2NodeClass. Make sure to upgrade your Custom Resource Definitions before proceeding with the upgrade. Failure to do so will result in Karpenter being unable to provision new nodes. +* Karpenter no longer updates the logger name when creating controller loggers. We now adhere to the controller-runtime standard, where the logger name will be set as `"logger": "controller"` always and the controller name will be stored in the structured value `"controller"` * Karpenter updated the NodeClass controller naming in the following way: `nodeclass` -> `nodeclass.status`, `nodeclass.hash`, `nodeclass.termination` +* Karpenter's NodeClaim status conditions no longer include the `severity` field ### Upgrading to `0.36.0`+ @@ -48,7 +96,7 @@ WHEN CREATING A NEW SECTION OF THE UPGRADE GUIDANCE FOR NEWER VERSIONS, ENSURE T {{% /alert %}} {{% alert title="Warning" color="warning" %}} - v0.36.x introduces update to drift that restricts rollback. When rolling back from >=v0.36.0, note that v0.32.9+, v0.33.4+, v0.34.5+, v0.35.4+ are the patch versions that support rollback. If Karpenter is rolled back to an older patch version, Karpenter can potentially drift all the nodes in the cluster. + v0.36.x introduces update to drift that restricts rollback. When rolling back from >=v0.36.0, note that v0.32.9+, v0.33.4+, v0.34.5+, v0.35.4+ are the patch versions that support rollback. If Karpenter is rolled back to an older patch version, Karpenter can potentially drift all the nodes in the cluster. {{% /alert %}} * Karpenter changed the name of the `karpenter_cloudprovider_instance_type_price_estimate` metric to `karpenter_cloudprovider_instance_type_offering_price_estimate` to align with the new `karpenter_cloudprovider_instance_type_offering_available` metric. The `region` label was also dropped from the metric, since this can be inferred from the environment that Karpenter is running in. @@ -76,7 +124,7 @@ The Ubuntu EKS optimized AMI has moved from 20.04 to 22.04 for Kubernetes 1.29+. * `Empty Expiration / Empty Drift / Empty Consolidation`: infinite parallelism * `Non-Empty Expiration / Non-Empty Drift / Single-Node Consolidation`: one node at a time * `Multi-Node Consolidation`: max 100 nodes -* To support Disruption Budgets, `0.34.0`+ includes critical changes to Karpenter's core controllers, which allows Karpenter to consider multiple batches of disrupting nodes simultaneously. This increases Karpenter's performance with the potential downside of higher CPU and memory utilization from the Karpenter pod. While the magnitude of this difference varies on a case-by-case basis, when upgrading to Karpenter `0.34.0`+, please note that you may need to increase the resources allocated to the Karpenter controller pods. +* To support Disruption Budgets, `0.34.0`+ includes critical changes to Karpenter's core controllers, which allows Karpenter to consider multiple batches of disrupting nodes simultaneously. This increases Karpenter's performance with the potential downside of higher CPU and memory utilization from the Karpenter pod. While the magnitude of this difference varies on a case-by-case basis, when upgrading to Karpenter `0.34.0`+, please note that you may need to increase the resources allocated to the Karpenter controller pods. * Karpenter now adds a default `podSecurityContext` that configures the `fsgroup: 65536` of volumes in the pod. If you are using sidecar containers, you should review if this configuration is compatible for them. You can disable this default `podSecurityContext` through helm by performing `--set podSecurityContext=null` when installing/upgrading the chart. * The `dnsPolicy` for the Karpenter controller pod has been changed back to the Kubernetes cluster default of `ClusterFirst`. Setting our `dnsPolicy` to `Default` (confusingly, this is not the Kubernetes cluster default) caused more confusion for any users running IPv6 clusters with dual-stack nodes or anyone running Karpenter with dependencies on cluster services (like clusters running service meshes). This change may be breaking for any users on Fargate or MNG who were allowing Karpenter to manage their in-cluster DNS service (`core-dns` on most clusters). If you still want the old behavior here, you can change the `dnsPolicy` to point to use `Default` by setting the helm value on install/upgrade with `--set dnsPolicy=Default`. More details on this issue can be found in the following Github issues: [#2186](https://github.com/aws/karpenter-provider-aws/issues/2186) and [#4947](https://github.com/aws/karpenter-provider-aws/issues/4947). * Karpenter now disallows `nodepool.spec.template.spec.resources` to be set. The webhook validation never allowed `nodepool.spec.template.spec.resources`. We are now ensuring that CEL validation also disallows `nodepool.spec.template.spec.resources` to be set. If you were previously setting the resources field on your NodePool, ensure that you remove this field before upgrading to the newest version of Karpenter or else updates to the resource may fail on the new version. @@ -104,9 +152,10 @@ Karpenter `0.32.0` introduces v1beta1 APIs, including _significant_ changes to t This version includes **dual support** for both alpha and beta APIs to ensure that you can slowly migrate your existing Provisioner, AWSNodeTemplate, and Machine alpha APIs to the newer NodePool, EC2NodeClass, and NodeClaim beta APIs. -Note that if you are rolling back after upgrading to `0.32.0`, note that `0.31.4` is the only version that supports handling rollback after you have deployed the v1beta1 APIs to your cluster. +Note that if you are rolling back after upgrading to `0.32.0`, note that __only__ versions `0.31.4` support handling rollback after you have deployed the v1beta1 APIs to your cluster. {{% /alert %}} +* Karpenter now uses `settings.InterruptionQueue` instead of `settings.aws.InterruptionQueueName` in its helm chart. The CLI argument also changed to `--interruption-queue`. * Karpenter now serves the webhook prometheus metrics server on port `8001`. If this port is already in-use on the pod or you are running in `hostNetworking` mode, you may need to change this port value. You can configure this port value through the `WEBHOOK_METRICS_PORT` environment variable or the `webhook.metrics.port` value if installing via Helm. * Karpenter now exposes the ability to disable webhooks through the `webhook.enabled=false` value. This value will disable the webhook server and will prevent any permissions, mutating or validating webhook configurations from being deployed to the cluster. * Karpenter now moves all logging configuration for the Zap logger into the `logConfig` values block. Configuring Karpenter logging with this mechanism _is_ deprecated and will be dropped at v1. Karpenter now only surfaces logLevel through the `logLevel` helm value. If you need more advanced configuration due to log parsing constraints, we recommend configuring your log parser to handle Karpenter's Zap JSON logging. diff --git a/website/content/en/preview/upgrading/v1-migration.md b/website/content/en/preview/upgrading/v1-migration.md new file mode 100644 index 000000000000..b18336cabacf --- /dev/null +++ b/website/content/en/preview/upgrading/v1-migration.md @@ -0,0 +1,413 @@ +--- +title: "v1 Migration" +linkTitle: "v1 Migration" +weight: 30 +description: > + Upgrade information for migrating to v1 +--- + +This migration guide is designed to help you migrate Karpenter from v1beta1 APIs to v1 (v0.33-v0.37). +Use this document as a reference to the changes that were introduced in this release and as a guide to how you need to update the manifests and other Karpenter objects you created in previous Karpenter releases. + +Before you begin upgrading to `v1.0.0`, you should know that: + +* Every Karpenter upgrade from pre-v1.0.0 versions must upgrade to minor version `v1.0.0`. +* You must be upgrading to `v1.0.0` from a version of Karpenter that only supports v1beta1 APIs, e.g. NodePools, NodeClaims, and NodeClasses (v0.33+). +* Karpenter `v1.0.0`+ supports Karpenter v1 and v1beta1 APIs and will not work with earlier Provisioner, AWSNodeTemplate or Machine v1alpha1 APIs. Do not upgrade to `v1.0.0`+ without first [upgrading to `0.32.x`]({{}}) or later and then upgrading to v0.33. +* Version `v1.0.0` adds [conversion webhooks](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/#webhook-conversion) to automatically pull the v1 API version of previously applied v1beta1 NodePools, EC2NodeClasses, and NodeClaims. Karpenter will stop serving the v1beta1 API version at v1.1.0 and will drop the conversion webhooks at that time. You will need to migrate all stored manifests to v1 API versions on Karpenter v1.0+. Keep in mind that this is a conversion and not dual support, which means that resources are updated in-place rather than migrated over from the previous version. +* If you need to rollback the upgrade to v1, you need to upgrade to a special patch version of the minor version you came from. For instance, if you came from v0.33.5, you'll need to downgrade back to v0.33.6. More details on how to do this in [Downgrading]({{}}). +* Validate that you are running at least Kubernetes 1.25. Use the [compatibility matrix]({{}}) to confirm you are on a supported Kubernetes version. + +See the [Changelog]({{}}) for details about actions you should take before upgrading to v1.0 or v1.1. + +## Upgrade Procedure + +Please read through the entire procedure before beginning the upgrade. There are major changes in this upgrade, so please evaluate the list of breaking changes before continuing. + +{{% alert title="Note" color="warning" %}} +The upgrade guide will first require upgrading to your latest patch version prior to upgrade to v1.0.0. This will be to allow the conversion webhooks to operate and minimize downtime of the Karpenter controller when requesting the Karpenter custom resources. +{{% /alert %}} + +1. Set environment variables for your cluster to upgrade to the latest patch version of the current Karpenter version you're running on: + + ```bash + export KARPENTER_NAMESPACE=kube-system + export KARPENTER_IAM_ROLE_ARN="arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/${CLUSTER_NAME}-karpenter" + export AWS_PARTITION="aws" # if you are not using standard partitions, you may need to configure to aws-cn / aws-us-gov + export CLUSTER_NAME="${USER}-karpenter-demo" + export AWS_REGION="us-west-2" + export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" + ``` + + +2. Determine the current Karpenter version: + ```bash + kubectl get pod -A | grep karpenter + kubectl describe pod -n "${KARPENTER_NAMESPACE}" karpenter-xxxxxxxxxx-xxxxx | grep Image: + ``` + Sample output: + ```bash + Image: public.ecr.aws/karpenter/controller:0.37.1@sha256:157f478f5db1fe999f5e2d27badcc742bf51cc470508b3cebe78224d0947674f + ``` + + The Karpenter version you are running must be between minor version `v0.33` and `v0.37`. To be able to roll back from Karpenter v1, you must rollback to on the following patch release versions for your minor version, which will include the conversion webhooks for a smooth rollback: + + * v0.37.1 + * v0.36.3 + * v0.35.6 + * v0.34.7 + * v0.33.6 + +3. Review for breaking changes between v0.33 and v0.37: If you are already running Karpenter v0.37.x, you can skip this step. If you are running an earlier Karpenter version, you need to review the [Upgrade Guide]({{}}) for each minor release. + +4. Set environment variables for upgrading to the latest patch version: + + ```bash + export KARPENTER_VERSION= + ``` + +6. Apply the latest patch version of your current minor version's Custom Resource Definitions (CRDs): + + ```bash + helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version "${KARPENTER_VERSION}" --namespace "${KARPENTER_NAMESPACE}" --create-namespace \ + --set webhook.enabled=true \ + --set webhook.serviceName=karpenter \ + --set webhook.serviceNamespace="${KARPENTER_NAMESPACE}" \ + --set webhook.port=8443 + ``` + + +7. Upgrade Karpenter to the latest patch version of your current minor version's. At the end of this step, conversion webhooks will run but will not convert any version. + + ```bash + # Service account annotation can be dropped when using pod identity + helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter --version ${KARPENTER_VERSION} --namespace "${KARPENTER_NAMESPACE}" --create-namespace \ + --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=${KARPENTER_IAM_ROLE_ARN} \ + --set settings.clusterName=${CLUSTER_NAME} \ + --set settings.interruptionQueue=${CLUSTER_NAME} \ + --set controller.resources.requests.cpu=1 \ + --set controller.resources.requests.memory=1Gi \ + --set controller.resources.limits.cpu=1 \ + --set controller.resources.limits.memory=1Gi \ + --set webhook.enabled=true \ + --set webhook.port=8443 \ + --wait + ``` + +8. Set environment variables for first upgrading to v1.0.0 + + ```bash + export KARPENTER_VERSION=1.0.0 + ``` + + +9. Update your existing policy using the following to the v1.0.0 controller policy: + Notable Changes to the IAM Policy include additional tag-scoping for the `eks:eks-cluster-name` tag for instances and instance profiles. + + ```bash + TEMPOUT=$(mktemp) + curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml > ${TEMPOUT} \ + && aws cloudformation deploy \ + --stack-name "Karpenter-${CLUSTER_NAME}" \ + --template-file "${TEMPOUT}" \ + --capabilities CAPABILITY_NAMED_IAM \ + --parameter-overrides "ClusterName=${CLUSTER_NAME}" + ``` + +10. Apply the v1.0.0 Custom Resource Definitions (CRDs): + + ```bash + helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version "${KARPENTER_VERSION}" --namespace "${KARPENTER_NAMESPACE}" --create-namespace \ + --set webhook.enabled=true \ + --set webhook.serviceName=karpenter \ + --set webhook.serviceNamespace="${KARPENTER_NAMESPACE}" \ + --set webhook.port=8443 + ``` + +11. Upgrade Karpenter to the new version. At the end of this step, conversion webhooks run to convert the Karpenter CRDs to v1. + + ```bash + # Service account annotion can be dropped when using pod identity + helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter --version ${KARPENTER_VERSION} --namespace "${KARPENTER_NAMESPACE}" --create-namespace \ + --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=${KARPENTER_IAM_ROLE_ARN} \ + --set settings.clusterName=${CLUSTER_NAME} \ + --set settings.interruptionQueue=${CLUSTER_NAME} \ + --set controller.resources.requests.cpu=1 \ + --set controller.resources.requests.memory=1Gi \ + --set controller.resources.limits.cpu=1 \ + --set controller.resources.limits.memory=1Gi \ + --wait + ``` + + {{% alert title="Note" color="warning" %}} + Karpenter has deprecated and moved a number of Helm values as part of the v1 release. Ensure that you upgrade to the newer version of these helm values during your migration to v1. You can find detail for all the settings that were moved in the [v1 Upgrade Reference]({{}}). + {{% /alert %}} + +12. Once upgraded, you won't need to roll your nodes to be compatible with v1.1.0, except if you have multiple NodePools with different `kubelet`s that are referencing the same EC2NodeClass. Karpenter has moved the `kubelet` to the EC2NodeClass in v1. NodePools with different `kubelet` referencing the same EC2NodeClass will be compatible with v1.0.0, but will not be in v1.1.0. + +When you have completed the migration to `1.0.0` CRDs, Karpenter will be able to serve both the `v1beta1` versions and the `v1` versions of NodePools, NodeClaims, and EC2NodeClasses. +The results of upgrading these CRDs include the following: + +* The storage version of these resources change to v1. After the upgrade, Karpenter starts converting these resources to v1 storage versions in real time. Users should experience no differences from this change. +* You are still able to GET and make updates using the v1beta1 versions, by for example doing `kubectl get nodepools.v1beta1.karpenter.sh`. + + +## Post upgrade considerations + +Your NodePool and EC2NodeClass objects are auto-converted to the new v1 storage version during the upgrade. Consider getting the latest versions of those objects to update any stored manifests where you were previously applying the v1beta1 version. + + * [NodePools]({{}}): Get the latest copy of your NodePool (`kubectl get nodepool default -o yaml > nodepool.yaml`) and review the [Changelog]({{}}) for changes to NodePool objects. Make modifications as needed. + * [EC2NodeClasses]({{}}): Get the latest copy of your EC2NodeClass (`kubectl get ec2nodeclass default -o yaml > ec2nodeclass.yaml`) and review the [Changelog]({{}}) for changes to EC2NodeClass objects. Make modifications as needed. + +When you are satisfied with your NodePool and EC2NodeClass files, apply them as follows: + +```bash +kubectl apply -f nodepool.yaml +kubectl apply -f ec2nodeclass.yaml +``` + +## Changelog +Refer to the [Full Changelog]({{}}) for more. + +Because Karpenter `v1.0.0` will run both `v1` and `v1beta1` versions of NodePools and EC2NodeClasses, you don't immediately have to upgrade the stored manifests that you have to v1. +However, in preparation for later Karpenter upgrades (which will not support `v1beta1`, review the following changes from v1beta1 to v1. + +Karpenter `v1.0.0` changes are divided into two different categories: those you must do before `1.0.0` upgrades and those you must do before `1.1.0` upgrades. + +### Changes required before upgrading to `v1.0.0` + +Apply the following changes to your NodePools and EC2NodeClasses, as appropriate, before upgrading them to v1. + +* **Deprecated annotations, labels and tags are removed for v1.0.0**: For v1, `karpenter.sh/do-not-consolidate` (annotation), `karpenter.sh/do-not-evict +(annotation)`, and `karpenter.sh/managed-by` (tag) all have support removed. +The `karpenter.sh/managed-by`, which currently stores the cluster name in its value, is replaced by `eks:eks-cluster-name`, to allow +for [EKS Pod Identity ABAC policies](https://docs.aws.amazon.com/eks/latest/userguide/pod-id-abac.html). + +* **Zap logging config removed**: Support for setting the Zap logging config was deprecated in beta and is now removed for v1. View the [Logging Configuration Section of the v1beta1 Migration Guide]({{}}) for more details. + +* **metadataOptions could break workloads**: If you have workload pods that are not using `hostNetworking`, the updated default `metadataOptions` could cause your containers to break when you apply new EC2NodeClasses on v1. + +* **Ubuntu AMIFamily Removed**: + + Support for automatic AMI selection and UserData generation for Ubuntu has been dropped with Karpenter `v1.0.0`. + To continue using Ubuntu AMIs you will need to specify an AMI using `amiSelectorTerms`. + + UserData generation can be achieved using the AL2 AMIFamily which has an identical UserData format. + However, compatibility is not guaranteed long-term and changes to either AL2 or Ubuntu's UserData format may introduce incompatibilities. + If this occurs, the Custom AMIFamily should be used for Ubuntu and UserData will need to be entirely maintained by the user. + + If you are upgrading to `v1.0.0` and already have v1beta1 Ubuntu EC2NodeClasses, all you need to do is specify `amiSelectorTerms` and Karpenter will translate your NodeClasses to the v1 equivalent (as shown below). + Failure to specify `amiSelectorTerms` will result in the EC2NodeClass and all referencing NodePools to show as NotReady, causing Karpenter to ignore these NodePools and EC2NodeClasses for Provisioning and Drift. + + ```yaml + # Original v1beta1 EC2NodeClass + version: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + spec: + amiFamily: Ubuntu + amiSelectorTerms: + - id: ami-foo + --- + # Conversion Webhook Output + version: karpenter.k8s.aws/v1 + kind: EC2NodeClass + metadata: + annotations: + compatibility.karpenter.k8s.aws/v1beta1-ubuntu: amiFamily,blockDeviceMappings + spec: + amiFamily: AL2 + amiSelectorTerms: + - id: ami-foo + blockDeviceMappings: + - deviceName: '/dev/sda1' + rootVolume: true + ebs: + encrypted: true + volumeType: gp3 + volumeSize: 20Gi + ``` + +* **amiSelectorTerms and amiFamily**: For v1, `amiFamily` is no longer required if you instead specify an `alias` in `amiSelectorTerms` in your `EC2NodeClass`. You need to update your `amiSelectorTerms` and `amiFamily` if you are using: + * A Custom amiFamily. You must ensure that the node you add the `karpenter.sh/unregistered:NoExecute` taint in your UserData. + * An Ubuntu AMI, as described earlier. + +### Before upgrading to `v1.1.0` + +Apply the following changes to your NodePools and EC2NodeClasses, as appropriate, before upgrading them to `v1.1.0` (though okay to make these changes for `1.0.0`) + +* **v1beta1 support gone**: In `v1.1.0`, v1beta1 is not supported. So you need to: + * Migrate all Karpenter yaml files [NodePools]({{}}), [EC2NodeClasses]({{}}) to v1. + * Know that all resources in the cluster also need to be on v1. It's possible (although unlikely) that some resources still may be stored as v1beta1 in ETCD if no writes had been made to them since the v1 upgrade. You could use a tool such as [kube-storage-version-migrator](https://github.com/kubernetes-sigs/kube-storage-version-migrator) to handle this. + * Know that you cannot rollback to v1beta1 once you have upgraded to `v1.1.0`. + +* **Kubelet Configuration**: If you have multiple NodePools pointing to the same EC2NodeClass that have different kubeletConfigurations, +then you have to manually add more EC2NodeClasses and point their NodePools to them. This will induce drift and you will have to roll your cluster. +If you have multiple NodePools pointing to the same EC2NodeClass, but they have the same configuration, then you can proceed with the migration +without having drift or having any additional NodePools or EC2NodeClasses configured. + +* **Remove kubelet annotation from NodePools**: During the upgrade process Karpenter will rely on the `compatibility.karpenter.sh/v1beta1-kubelet-conversion` annotation to determine whether to use the v1beta1 NodePool kubelet configuration or the v1 EC2NodeClass kubelet configuration. The `compatibility.karpenter.sh/v1beta1-kubelet-conversion` NodePool annotation takes precedence over the EC2NodeClass Kubelet configuration when launching nodes. Remove the kubelet-configuration annotation (`compatibility.karpenter.sh/v1beta1-kubelet-conversion`) from your NodePools once you have migrated kubelet from the NodePool to the EC2NodeClass. + +Keep in mind that rollback, without replacing the Karpenter nodes, will not be supported to an earlier version of Karpenter once the annotation is removed. This annotation is only used to support the kubelet configuration migration path, but will not be supported in v1.1. + +### Downgrading + +Once the Karpenter CRDs are upgraded to v1, conversion webhooks are needed to help convert APIs that are stored in etcd from v1 to v1beta1. Also changes to the CRDs will need to at least include the latest version of the CRD in this case being v1. The patch versions of the v1beta1 Karpenter controller that include the conversion wehooks include: + +* v0.37.1 +* v0.36.3 +* v0.35.6 +* v0.34.7 +* v0.33.6 + +{{% alert title="Note" color="warning" %}} +When rolling back from v1, Karpenter will not retain data that was only valid in v1 APIs. For instance, if you were upgrading from v0.33.5 to v1, updated the `NodePool.Spec.Disruption.Budgets` field and then rolled back to v0.33.6, Karpenter would not retain the `NodePool.Spec.Disruption.Budgets` field, as that was introduced in v0.34.x. If you are configuring the kubelet field, and have removed the `compatibility.karpenter.sh/v1beta1-kubelet-conversion` annotation, rollback is not supported without replacing your nodes between EC2NodeClass and NodePool. +{{% /alert %}} + +{{% alert title="Note" color="warning" %}} +Since both v1beta1 and v1 will be served, `kubectl` will default to returning the `v1` version of your CRDs. To interact with the v1beta1 version of your CRDs, you'll need to add the full resource path (including api version) into `kubectl` calls. For example: `k get nodeclaim.v1beta1.karpenter.sh` +{{% /alert %}} + +1. Set environment variables + +```bash +export KARPENTER_NAMESPACE="kube-system" +export KARPENTER_VERSION="" +export KARPENTER_IAM_ROLE_ARN="arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/${CLUSTER_NAME}-karpenter" +export CLUSTER_NAME="" +export TEMPOUT="$(mktemp)" +``` + +{{% alert title="Warning" color="warning" %}} +If you open a new shell to run steps in this procedure, you need to set some or all of the environment variables again. +To remind yourself of these values, type: + +```bash +echo "${KARPENTER_NAMESPACE}" "${KARPENTER_VERSION}" "${CLUSTER_NAME}" "${TEMPOUT}" +``` + +{{% /alert %}} + +2. Rollback the Karpenter Policy + +```bash +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml > ${TEMPOUT} \ + && aws cloudformation deploy \ + --stack-name "Karpenter-${CLUSTER_NAME}" \ + --template-file "${TEMPOUT}" \ + --capabilities CAPABILITY_NAMED_IAM \ + --parameter-overrides "ClusterName=${CLUSTER_NAME}" +``` + +3. Rollback the CRDs + +```bash +helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version "${KARPENTER_VERSION}" --namespace "${KARPENTER_NAMESPACE}" --create-namespace \ + --set webhook.enabled=true \ + --set webhook.serviceName=karpenter \ + --set webhook.serviceNamespace="${KARPENTER_NAMESPACE}" \ + --set webhook.port=8443 +``` + +4. Rollback the Karpenter Controller + +```bash +# Service account annotation can be dropped when using pod identity +helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter --version ${KARPENTER_VERSION} --namespace "${KARPENTER_NAMESPACE}" --create-namespace \ + --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=${KARPENTER_IAM_ROLE_ARN} \ + --set settings.clusterName=${CLUSTER_NAME} \ + --set settings.interruptionQueue=${CLUSTER_NAME} \ + --set controller.resources.requests.cpu=1 \ + --set controller.resources.requests.memory=1Gi \ + --set controller.resources.limits.cpu=1 \ + --set controller.resources.limits.memory=1Gi \ + --set webhook.enabled=true \ + --set webhook.port=8443 \ + --wait +``` + +Karpenter should now be pulling and operating against the v1beta1 APIVersion as it was prior to the upgrade + +## Full Changelog +* Features: + * AMI Selector Terms has a new Alias field which can only be set by itself in `EC2NodeClass.Spec.AMISelectorTerms` + * Disruption Budgets by Reason was added to `NodePool.Spec.Disruption.Budgets` + * TerminationGracePeriod was added to `NodePool.Spec.Template.Spec`. + * LOG_OUTPUT_PATHS and LOG_ERROR_OUTPUT_PATHS environment variables added +* API Rename: NodePool’s ConsolidationPolicy `WhenUnderutilized` is now renamed to `WhenEmptyOrUnderutilized` +* Behavior Changes: + * Expiration is now forceful and begins draining as soon as it’s expired. Karpenter does not wait for replacement capacity to be available before draining, but will start provisioning a replacement as soon as the node is expired and begins draining. + * Karpenter's generated NodeConfig now takes precedence when generating UserData with the AL2023 `amiFamily`. If you're setting any values managed by Karpenter in your AL2023 UserData, configure these through Karpenter natively (e.g. kubelet configuration fields). + * Karpenter now adds a `karpenter.sh/unregistered:NoExecute` taint to nodes in injected UserData when using alias in AMISelectorTerms or non-Custom AMIFamily. When using `amiFamily: Custom`, users will need to add this taint into their UserData, where Karpenter will automatically remove it when provisioning nodes. +* API Moves: + * ExpireAfter has moved from the `NodePool.Spec.Disruption` block to `NodePool.Spec.Template.Spec`, and is now a drift-able field. + * `Kubelet` was moved to the EC2NodeClass from the NodePool. +* RBAC changes: added `delete pods` | added `get, patch crds` | added `update nodes` | removed `create nodes` +* Breaking API (Manual Migration Needed): + * Ubuntu is dropped as a first class supported AMI Family + * `karpenter.sh/do-not-consolidate` (annotation), `karpenter.sh/do-not-evict` (annotation), and `karpenter.sh/managed-by` (tag) are all removed. `karpenter.sh/managed-by`, which currently stores the cluster name in its value, will be replaced by eks:eks-cluster-name + * The taint used to mark nodes for disruption and termination changed from `karpenter.sh/disruption=disrupting:NoSchedule` to `karpenter.sh/disrupted:NoSchedule`. It is not recommended to tolerate this taint, however, if you were tolerating it in your applications, you'll need to adjust your taints to reflect this. +* Environment Variable Changes: + * Environment Variable Changes + * LOGGING_CONFIG, ASSUME_ROLE_ARN, ASSUME_ROLE_DURATION Dropped + * LEADER_ELECT renamed to DISABLE_LEADER_ELECTION + * `FEATURE_GATES.DRIFT=true` was dropped and promoted to Stable, and cannot be disabled. + * Users currently opting out of drift, disabling the drift feature flag will no longer be able to do so. +* Defaults changed: + * API: Karpenter will drop support for IMDS access from containers by default on new EC2NodeClasses by updating the default of `httpPutResponseHopLimit` from 2 to 1. + * API: ConsolidateAfter is required. Users couldn’t set this before with ConsolidationPolicy: WhenUnderutilized, where this is now required. Users can set it to 0 to have the same behavior as in v1beta1. + * API: All `NodeClassRef` fields are now all required, and apiVersion has been renamed to group + * API: AMISelectorTerms are required. Setting an Alias cannot be done with any other type of term, and must match the AMI Family that's set or be Custom. + * Helm: Deployment spec TopologySpreadConstraint to have required zonal spread over preferred. Users who had one node running their Karpenter deployments need to either: + * Have two nodes in different zones to ensure both Karpenter replicas schedule + * Scale down their Karpenter replicas from 2 to 1 in the helm chart + * Edit and relax the topology spread constraint in their helm chart from DoNotSchedule to ScheduleAnyway + * Helm/Binary: `controller.METRICS_PORT` default changed back to 8080 + +### Updated metrics + +Changes to Karpenter metrics from v1beta1 to v1 are shown in the following tables. + +This table shows metrics names that changed from v1beta1 to v1: + +| Metric type | v1beta1 metrics name | new v1 metrics name | +|--|--|--| +| Node | karpenter_nodes_termination_time_seconds | karpenter_nodes_termination_duration_seconds | +| Node | karpenter_nodes_terminated | karpenter_nodes_terminated_total | +| Node | karpenter_nodes_leases_deleted | karpenter_nodes_leases_deleted_total | +| Node | karpenter_nodes_created | karpenter_nodes_created_total | +| Pod | karpenter_pods_startup_time_seconds | karpenter_pods_startup_duration_seconds | +| Disruption | karpenter_disruption_replacement_nodeclaim_failures_total | karpenter_voluntary_disruption_queue_failures_total | +| Disruption | karpenter_disruption_evaluation_duration_seconds | karpenter_voluntary_disruption_decision_evaluation_duration_seconds | +| Disruption | karpenter_disruption_eligible_nodes | karpenter_voluntary_disruption_eligible_nodes | +| Disruption | karpenter_disruption_consolidation_timeouts_total | karpenter_voluntary_disruption_consolidation_timeouts_total | +| Disruption | karpenter_disruption_budgets_allowed_disruptions | karpenter_nodepools_allowed_disruptions | +| Disruption | karpenter_disruption_actions_performed_total | karpenter_voluntary_disruption_decisions_total | +| Provisioner | karpenter_provisioner_scheduling_simulation_duration_seconds | karpenter_scheduler_scheduling_duration_seconds | +| Provisioner | karpenter_provisioner_scheduling_queue_depth | karpenter_scheduler_queue_depth | +| Interruption | karpenter_interruption_received_messages | karpenter_interruption_received_messages_total | +| Interruption | karpenter_interruption_deleted_messages | karpenter_interruption_deleted_messages_total | +| Interruption | karpenter_interruption_message_latency_time_seconds | karpenter_interruption_message_queue_duration_seconds | +| NodePool | karpenter_nodepool_usage | karpenter_nodepools_usage | +| NodePool | karpenter_nodepool_limit | karpenter_nodepools_limit | +| NodeClaim | karpenter_nodeclaims_terminated | karpenter_nodeclaims_terminated_total | +| NodeClaim | karpenter_nodeclaims_disrupted | karpenter_nodeclaims_disrupted_total | +| NodeClaim | karpenter_nodeclaims_created | karpenter_nodeclaims_created_total | + +This table shows v1beta1 metrics that were dropped for v1: + +| Metric type | Metric dropped for v1 | +|--|--| +| Disruption | karpenter_disruption_replacement_nodeclaim_initialized_seconds | +| Disruption | karpenter_disruption_queue_depth | +| Disruption | karpenter_disruption_pods_disrupted_total | +| | karpenter_consistency_errors | +| NodeClaim | karpenter_nodeclaims_registered | +| NodeClaim | karpenter_nodeclaims_launched | +| NodeClaim | karpenter_nodeclaims_initialized | +| NodeClaim | karpenter_nodeclaims_drifted | +| Provisioner | karpenter_provisioner_scheduling_duration_seconds | +| Interruption | karpenter_interruption_actions_performed | + +{{% alert title="Note" color="warning" %}} +Karpenter now waits for the underlying instance to be completely terminated before deleting a node and orchestrates this by emitting `NodeClaimNotFoundError`. With this change we expect to see an increase in the `NodeClaimNotFoundError`. Customers can filter out this error by label in order to get accurate values for `karpenter_cloudprovider_errors_total` metric. Use this Prometheus filter expression - `({controller!="node.termination"} or {controller!="nodeclaim.termination"}) and {error!="NodeClaimNotFoundError"}`. +{{% /alert %}} diff --git a/website/content/en/v0.32/concepts/_index.md b/website/content/en/v0.32/concepts/_index.md index 67d573135369..e7f8ce4cb500 100755 --- a/website/content/en/v0.32/concepts/_index.md +++ b/website/content/en/v0.32/concepts/_index.md @@ -29,7 +29,7 @@ Once privileges are in place, Karpenter is deployed with a Helm chart. ### Configuring NodePools -Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. To configure Karpenter, you create [NodePools]({{}}) that define how Karpenter manages unschedulable pods and configures nodes. You will also define behaviors for your NodePools, capturing details like how Karpenter handles disruption of nodes and setting limits and weights for each NodePool +Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. To configure Karpenter, you create [NodePools]({{}}) that define how Karpenter manages unschedulable pods and configures nodes. You will also define behaviors for your NodePools, capturing details like how Karpenter handles disruption of nodes and setting limits and weights for each NodePool. Here are some things to know about Karpenter's NodePools: diff --git a/website/content/en/v0.32/concepts/disruption.md b/website/content/en/v0.32/concepts/disruption.md index 408e591147a0..b7ebec2b83df 100644 --- a/website/content/en/v0.32/concepts/disruption.md +++ b/website/content/en/v0.32/concepts/disruption.md @@ -174,7 +174,7 @@ If you require handling for Spot Rebalance Recommendations, you can use the [AWS Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). -To enable interruption handling, configure the `--interruption-queue-name` CLI argument with the name of the interruption queue provisioned to handle interruption events. +To enable interruption handling, configure the `--interruption-queue` CLI argument with the name of the interruption queue provisioned to handle interruption events. ## Controls diff --git a/website/content/en/v0.32/concepts/nodeclasses.md b/website/content/en/v0.32/concepts/nodeclasses.md index a5fbbd0636cb..d8e6a528787a 100644 --- a/website/content/en/v0.32/concepts/nodeclasses.md +++ b/website/content/en/v0.32/concepts/nodeclasses.md @@ -153,7 +153,7 @@ status: # Generated instance profile name from "role" instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" ``` -Refer to the [NodePool docs]({{}}) for settings applicable to all providers. To explore various `EC2NodeClass` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). +Refer to the [NodePool docs]({{}}) for settings applicable to all providers. To explore various `EC2NodeClass` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/v0.32.0/examples/v1beta1/). ## spec.amiFamily @@ -396,6 +396,10 @@ spec: AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). **When you specify `amiSelectorTerms`, you fully override the default AMIs that are selected on by your EC2NodeClass [`amiFamily`]({{< ref "#specamifamily" >}}).** +{{% alert title="Note" color="primary" %}} +[`amiFamily`]({{< ref "#specamifamily" >}}) determines the bootstrapping mode, while `amiSelectorTerms` specifies specific AMIs to be used. Therefore, you need to ensure consistency between [`amiFamily`]({{< ref "#specamifamily" >}}) and `amiSelectorTerms` to avoid conflicts during bootstrapping. +{{% /alert %}} + This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. ```yaml @@ -666,7 +670,7 @@ spec: chown -R ec2-user ~ec2-user/.ssh ``` -For more examples on configuring fields for different AMI families, see the [examples here](https://github.com/aws/karpenter/blob/main/examples/v1beta1). +For more examples on configuring fields for different AMI families, see the [examples here](https://github.com/aws/karpenter/blob/v0.32.0/examples/v1beta1/). Karpenter will merge the userData you specify with the default userData for that AMIFamily. See the [AMIFamily]({{< ref "#specamifamily" >}}) section for more details on these defaults. View the sections below to understand the different merge strategies for each AMIFamily. diff --git a/website/content/en/v0.32/concepts/nodepools.md b/website/content/en/v0.32/concepts/nodepools.md index 607311c5ef6e..3a8824f16bba 100644 --- a/website/content/en/v0.32/concepts/nodepools.md +++ b/website/content/en/v0.32/concepts/nodepools.md @@ -22,7 +22,7 @@ Here are things you should know about NodePools: * If Karpenter encounters a startup taint in the NodePool it will be applied to nodes that are provisioned, but pods do not need to tolerate the taint. Karpenter assumes that the taint is temporary and some other system will remove the taint. * It is recommended to create NodePools that are mutually exclusive. So no Pod should match multiple NodePools. If multiple NodePools are matched, Karpenter will use the NodePool with the highest [weight](#specweight). -For some example `NodePool` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). +For some example `NodePool` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/v0.32.0/examples/v1beta1/). ```yaml apiVersion: karpenter.sh/v1beta1 @@ -140,7 +140,7 @@ spec: # You can choose to disable expiration entirely by setting the string value 'Never' here expireAfter: 720h - # Resource limits constrain the total size of the cluster. + # Resource limits constrain the total size of the pool. # Limits prevent Karpenter from creating new instances once the limit is exceeded. limits: cpu: "1000" @@ -243,6 +243,10 @@ spec: {{% /alert %}} +{{% alert title="Note" color="primary" %}} +There is currently a limit of 30 on the total number of requirements on both the NodePool and the NodeClaim. It's important to note that `spec.template.metadata.labels` are also propagated as requirements on the NodeClaim when it's created, meaning that you can't have more than 30 requirements and labels combined set on your NodePool. +{{% /alert %}} + ## spec.template.spec.nodeClassRef This field points to the Cloud Provider NodeClass resource. Learn more about [EC2NodeClasses]({{}}). diff --git a/website/content/en/v0.32/concepts/scheduling.md b/website/content/en/v0.32/concepts/scheduling.md index 466b54910454..147f4e00896e 100755 --- a/website/content/en/v0.32/concepts/scheduling.md +++ b/website/content/en/v0.32/concepts/scheduling.md @@ -104,8 +104,6 @@ Refer to general [Kubernetes GPU](https://kubernetes.io/docs/tasks/manage-gpus/s You must enable Pod ENI support in the AWS VPC CNI Plugin before enabling Pod ENI support in Karpenter. Please refer to the [Security Groups for Pods documentation](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) for instructions. {{% /alert %}} -Now that Pod ENI support is enabled in the AWS VPC CNI Plugin, you can enable Pod ENI support in Karpenter by setting the `settings.aws.enablePodENI` Helm chart value to `true`. - Here is an example of a pod-eni resource defined in a deployment manifest: ``` spec: @@ -173,6 +171,9 @@ requirements: - key: user.defined.label/type operator: Exists ``` +{{% alert title="Note" color="primary" %}} +There is currently a limit of 30 on the total number of requirements on both the NodePool and the NodeClaim. It's important to note that `spec.template.metadata.labels` are also propagated as requirements on the NodeClaim when it's created, meaning that you can't have more than 30 requirements and labels combined set on your NodePool. +{{% /alert %}} #### Node selectors @@ -280,7 +281,7 @@ spec: - p3 taints: - key: nvidia.com/gpu - value: true + value: "true" effect: "NoSchedule" ``` @@ -625,19 +626,73 @@ If using Gt/Lt operators, make sure to use values under the actual label values The `Exists` operator can be used on a NodePool to provide workload segregation across nodes. ```yaml -... -requirements: -- key: company.com/team - operator: Exists +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +spec: + template: + spec: + requirements: + - key: company.com/team + operator: Exists ... ``` -With the requirement on the NodePool, workloads can optionally specify a custom value as a required node affinity or node selector. Karpenter will then label the nodes it launches for these pods which prevents `kube-scheduler` from scheduling conflicting pods to those nodes. This provides a way to more dynamically isolate workloads without requiring a unique NodePool for each workload subset. +With this requirement on the NodePool, workloads can specify the same key (e.g. `company.com/team`) with custom values (e.g. `team-a`, `team-b`, etc.) as a required `nodeAffinity` or `nodeSelector`. Karpenter will then apply the key/value pair to nodes it launches dynamically based on the pod's node requirements. + +If each set of pods that can schedule with this NodePool specifies this key in its `nodeAffinity` or `nodeSelector`, you can isolate pods onto different nodes based on their values. This provides a way to more dynamically isolate workloads without requiring a unique NodePool for each workload subset. + +For example, providing the following `nodeSelectors` would isolate the pods for each of these deployments on different nodes. + +#### Team A Deployment ```yaml -nodeSelector: - company.com/team: team-a +apiVersion: v1 +kind: Deployment +metadata: + name: team-a-deployment +spec: + replicas: 5 + template: + spec: + nodeSelector: + company.com/team: team-a ``` + +#### Team A Node + +```yaml +apiVersion: v1 +kind: Node +metadata: + labels: + company.com/team: team-a +``` + +#### Team B Deployment + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: team-b-deployment +spec: + replicas: 5 + template: + spec: + nodeSelector: + company.com/team: team-b +``` + +#### Team B Node + +```yaml +apiVersion: v1 +kind: Node +metadata: + labels: + company.com/team: team-b +``` + {{% alert title="Note" color="primary" %}} If a workload matches the NodePool but doesn't specify a label, Karpenter will generate a random label for the node. {{% /alert %}} diff --git a/website/content/en/v0.32/faq.md b/website/content/en/v0.32/faq.md index eb3a46e16351..89fba5ca28fb 100644 --- a/website/content/en/v0.32/faq.md +++ b/website/content/en/v0.32/faq.md @@ -14,7 +14,7 @@ See [Configuring NodePools]({{< ref "./concepts/#configuring-nodepools" >}}) for AWS is the first cloud provider supported by Karpenter, although it is designed to be used with other cloud providers as well. ### Can I write my own cloud provider for Karpenter? -Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.32.9/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. +Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.32.10/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. ### What operating system nodes does Karpenter deploy? When using v1beta1 APIs, Karpenter uses the OS defined by the [AMI Family in your EC2NodeClass]({{< ref "./concepts/nodeclasses#specamifamily" >}}). @@ -27,7 +27,7 @@ Karpenter has multiple mechanisms for configuring the [operating system]({{< ref Karpenter is flexible to multi-architecture configurations using [well known labels]({{< ref "./concepts/scheduling/#supported-labels">}}). ### What RBAC access is required? -All the required RBAC rules can be found in the helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/role.yaml) files for details. +All the required RBAC rules can be found in the helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.32.10/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.32.10/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.32.10/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.32.10/charts/karpenter/templates/role.yaml) files for details. ### Can I run Karpenter outside of a Kubernetes cluster? Yes, as long as the controller has network and IAM/RBAC access to the Kubernetes API and your provider API. @@ -207,15 +207,15 @@ For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgra ### How do I upgrade an EKS Cluster with Karpenter? -When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift currently needs to be enabled using a [feature gate]({{}}). - {{% alert title="Note" color="primary" %}} -Karpenter's default [EC2NodeClass `amiFamily` configuration]({{}}) uses the latest EKS Optimized AL2 AMI for the same major and minor version as the EKS cluster's control plane, meaning that an upgrade of the control plane will cause Karpenter to auto-discover the new AMIs for that version. +Karpenter recommends that you always validate AMIs in your lower environments before using them in production environments. Read [Managing AMIs]({{}}) to understand best practices about upgrading your AMIs. -If using a custom AMI, you will need to trigger the rollout of this new worker node image through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. +If using a custom AMI, you will need to trigger the rollout of new worker node images through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. {{% /alert %}} -Start by [upgrading the EKS Cluster control plane](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). After the EKS Cluster upgrade completes, Karpenter's Drift feature will detect that the Karpenter-provisioned nodes are using EKS Optimized AMIs for the previous cluster version, and [automatically cordon, drain, and replace those nodes]({{}}). To support pods moving to new nodes, follow Kubernetes best practices by setting appropriate pod [Resource Quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/), and using [Pod Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (PDB). Karpenter's Drift feature will spin up replacement nodes based on the pod resource requests, and will respect the PDBs when deprovisioning nodes. +Karpenter's default behavior will upgrade your nodes when you've upgraded your Amazon EKS Cluster. Karpenter will [drift]({{}}) nodes to stay in-sync with the EKS control plane version. Drift is enabled by default starting in `v0.33`. This means that as soon as your cluster is upgraded, Karpenter will auto-discover the new AMIs for that version. + +Start by [upgrading the EKS Cluster control plane](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). After the EKS Cluster upgrade completes, Karpenter will Drift and disrupt the Karpenter-provisioned nodes using EKS Optimized AMIs for the previous cluster version by first spinning up replacement nodes. Karpenter respects [Pod Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (PDB), and automatically [replaces, cordons, and drains those nodes]({{}}). To best support pods moving to new nodes, follow Kubernetes best practices by setting appropriate pod [Resource Quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/) and using PDBs. ## Interruption Handling @@ -227,7 +227,7 @@ Karpenter's native interruption handling offers two main benefits over the stand 1. You don't have to manage and maintain a separate component to exclusively handle interruption events. 2. Karpenter's native interruption handling coordinates with other deprovisioning so that consolidation, expiration, etc. can be aware of interruption events and vice-versa. -### Why am I receiving QueueNotFound errors when I set `--interruption-queue-name`? +### Why am I receiving QueueNotFound errors when I set `--interruption-queue`? Karpenter requires a queue to exist that receives event messages from EC2 and health services in order to handle interruption messages properly for nodes. Details on the types of events that Karpenter handles can be found in the [Interruption Handling Docs]({{< ref "./concepts/disruption/#interruption" >}}). diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/_index.md index d03eb9c5a546..22a8524af7f8 100644 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/_index.md +++ b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/_index.md @@ -45,7 +45,7 @@ After setting up the tools, set the Karpenter and Kubernetes version: ```bash export KARPENTER_NAMESPACE=karpenter -export KARPENTER_VERSION=v0.32.9 +export KARPENTER_VERSION=v0.32.10 export K8S_VERSION=1.28 ``` @@ -79,6 +79,9 @@ The following cluster configuration will: {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh" language="bash"%}} +Unless your AWS account has already onboarded to EC2 Spot, you will need to create the service linked role to +avoid the [`ServiceLinkedRoleCreationNotPermitted` error]({{}}). + {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh" language="bash"%}} {{% alert title="Windows Support Notice" color="warning" %}} @@ -197,6 +200,19 @@ Karpenter (controller and webhook deployment) container images must be in or cop {{% alert title="Note" color="primary" %}} +There is currently no VPC private endpoint for the [IAM API](https://docs.aws.amazon.com/IAM/latest/APIReference/welcome.html). As a result, you cannot use the default `spec.role` field in your `EC2NodeClass`. Instead, you need to provision and manage an instance profile manually and then specify Karpenter to use this instance profile through the `spec.instanceProfile` field. + +You can provision an instance profile manually and assign a Node role to it by calling the following command + +```bash +aws iam create-instance-profile --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" +aws iam add-role-to-instance-profile --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" --role-name "KarpenterNodeRole-${CLUSTER_NAME}" +``` + +{{% /alert %}} + +{{% alert title="Note" color="primary" %}} + There is currently no VPC private endpoint for the [Price List Query API](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/using-price-list-query-api.html). As a result, pricing data can go stale over time. By default, Karpenter ships a static price list that is updated when each binary is released. Failed requests for pricing data will result in the following error messages diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/cloudformation.yaml b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/cloudformation.yaml index 967e703fb9e2..29fb92ca3aaa 100644 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/cloudformation.yaml +++ b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/cloudformation.yaml @@ -196,7 +196,7 @@ Resources: { "Sid": "AllowScopedInstanceProfileCreationActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:CreateInstanceProfile" ], @@ -213,7 +213,7 @@ Resources: { "Sid": "AllowScopedInstanceProfileTagActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:TagInstanceProfile" ], @@ -233,7 +233,7 @@ Resources: { "Sid": "AllowScopedInstanceProfileActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:AddRoleToInstanceProfile", "iam:RemoveRoleFromInstanceProfile", @@ -252,7 +252,7 @@ Resources: { "Sid": "AllowInstanceProfileReadActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": "iam:GetInstanceProfile" }, { @@ -284,6 +284,14 @@ Resources: - sqs.amazonaws.com Action: sqs:SendMessage Resource: !GetAtt KarpenterInterruptionQueue.Arn + - Sid: DenyHTTP + Effect: Deny + Action: sqs:* + Resource: !GetAtt KarpenterInterruptionQueue.Arn + Condition: + Bool: + aws:SecureTransport: false + Principal: "*" ScheduledChangeRule: Type: 'AWS::Events::Rule' Properties: diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh index c17584cec718..0854100d516c 100755 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh +++ b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh @@ -14,12 +14,18 @@ spec: app: inflate spec: terminationGracePeriodSeconds: 0 + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 containers: - - name: inflate - image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 - resources: - requests: - cpu: 1 + - name: inflate + image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 + resources: + requests: + cpu: 1 + securityContext: + allowPrivilegeEscalation: false EOF kubectl scale deployment inflate --replicas 5 diff --git a/website/content/en/v0.32/getting-started/migrating-from-cas/_index.md b/website/content/en/v0.32/getting-started/migrating-from-cas/_index.md index 9c7e128cbff4..83134c7c83a9 100644 --- a/website/content/en/v0.32/getting-started/migrating-from-cas/_index.md +++ b/website/content/en/v0.32/getting-started/migrating-from-cas/_index.md @@ -92,7 +92,7 @@ One for your Karpenter node role and one for your existing node group. First set the Karpenter release you want to deploy. ```bash -export KARPENTER_VERSION=v0.32.9 +export KARPENTER_VERSION=v0.32.10 ``` We can now generate a full Karpenter deployment yaml from the helm chart. @@ -117,7 +117,6 @@ affinity: - matchExpressions: - key: karpenter.sh/nodepool operator: DoesNotExist - - matchExpressions: - key: eks.amazonaws.com/nodegroup operator: In values: @@ -133,7 +132,7 @@ Now that our deployment is ready we can create the karpenter namespace, create t ## Create default NodePool -We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.32.9/examples/v1beta1) for specific needs. +We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.32.10/examples/v1beta1) for specific needs. {{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh" language="bash" %}} diff --git a/website/content/en/v0.32/reference/cloudformation.md b/website/content/en/v0.32/reference/cloudformation.md index eabe68010866..f62bbf238154 100644 --- a/website/content/en/v0.32/reference/cloudformation.md +++ b/website/content/en/v0.32/reference/cloudformation.md @@ -17,7 +17,7 @@ These descriptions should allow you to understand: To download a particular version of `cloudformation.yaml`, set the version and use `curl` to pull the file to your local system: ```bash -export KARPENTER_VERSION=v0.32.9 +export KARPENTER_VERSION=v0.32.10 curl https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > cloudformation.yaml ``` @@ -350,14 +350,14 @@ This gives EC2 permission explicit permission to use the `KarpenterNodeRole-${Cl #### AllowScopedInstanceProfileCreationActions The AllowScopedInstanceProfileCreationActions Sid gives the Karpenter controller permission to create a new instance profile with [`iam:CreateInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_CreateInstanceProfile.html), -provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName` set to owned and is made in the current region. +provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName}` set to owned and is made in the current region. Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures that Karpenter can generate instance profiles on your behalf based on roles specified in your `EC2NodeClasses` that you use to configure Karpenter. ```json { "Sid": "AllowScopedInstanceProfileCreationActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:CreateInstanceProfile" ], @@ -382,7 +382,7 @@ Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures t { "Sid": "AllowScopedInstanceProfileTagActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:TagInstanceProfile" ], @@ -405,14 +405,14 @@ Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures t #### AllowScopedInstanceProfileActions The AllowScopedInstanceProfileActions Sid gives the Karpenter controller permission to perform [`iam:AddRoleToInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_AddRoleToInstanceProfile.html), [`iam:RemoveRoleFromInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_RemoveRoleFromInstanceProfile.html), and [`iam:DeleteInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_DeleteInstanceProfile.html) actions, -provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName` set to owned and is made in the current region. +provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName}` set to owned and is made in the current region. Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This permission is further enforced by the `iam:PassRole` permission. If Karpenter attempts to add a role to an instance profile that it doesn't have `iam:PassRole` permission on, that call will fail. Therefore, if you configure Karpenter to use a new role through the `EC2NodeClass`, ensure that you also specify that role within your `iam:PassRole` permission. ```json { "Sid": "AllowScopedInstanceProfileActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:AddRoleToInstanceProfile", "iam:RemoveRoleFromInstanceProfile", @@ -438,7 +438,7 @@ The AllowInstanceProfileActions Sid gives the Karpenter controller permission to { "Sid": "AllowInstanceProfileReadActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": "iam:GetInstanceProfile" } ``` @@ -499,6 +499,7 @@ KarpenterInterruptionQueue: The Karpenter interruption queue policy is created to allow AWS services that we want to receive instance notifications from to push notification messages to the queue. The [AWS::SQS::QueuePolicy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sqs-queuepolicy.html) resource here applies `EC2InterruptionPolicy` to the `KarpenterInterruptionQueue`. The policy allows [sqs:SendMessage](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html) actions to `events.amazonaws.com` and `sqs.amazonaws.com` services. It also allows the `GetAtt` function to get attributes from `KarpenterInterruptionQueue.Arn`. +Additionally, it only allows access to the queue using encrypted connections over HTTPS (TLS) to adhere to [Amazon SQS Security Best Practices](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-security-best-practices.html#enforce-encryption-data-in-transit). ```yaml KarpenterInterruptionQueuePolicy: @@ -516,6 +517,14 @@ KarpenterInterruptionQueuePolicy: - sqs.amazonaws.com Action: sqs:SendMessage Resource: !GetAtt KarpenterInterruptionQueue.Arn + - Sid: DenyHTTP + Effect: Deny + Action: sqs:* + Resource: !GetAtt KarpenterInterruptionQueue.Arn + Condition: + Bool: + aws:SecureTransport: false + Principal: "*" ``` ### Rules diff --git a/website/content/en/v0.32/reference/threat-model.md b/website/content/en/v0.32/reference/threat-model.md index 0e40d8a4669d..fb907195c32e 100644 --- a/website/content/en/v0.32/reference/threat-model.md +++ b/website/content/en/v0.32/reference/threat-model.md @@ -31,11 +31,11 @@ A Cluster Developer has the ability to create pods via `Deployments`, `ReplicaSe Karpenter has permissions to create and manage cloud instances. Karpenter has Kubernetes API permissions to create, update, and remove nodes, as well as evict pods. For a full list of the permissions, see the RBAC rules in the helm chart template. Karpenter also has AWS IAM permissions to create instances with IAM roles. -* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/aggregate-clusterrole.yaml) -* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/clusterrole-core.yaml) -* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/clusterrole.yaml) -* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/rolebinding.yaml) -* [role.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/role.yaml) +* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.32.10/charts/karpenter/templates/aggregate-clusterrole.yaml) +* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.32.10/charts/karpenter/templates/clusterrole-core.yaml) +* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.32.10/charts/karpenter/templates/clusterrole.yaml) +* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.32.10/charts/karpenter/templates/rolebinding.yaml) +* [role.yaml](https://github.com/aws/karpenter/blob/v0.32.10/charts/karpenter/templates/role.yaml) ## Assumptions diff --git a/website/content/en/v0.32/troubleshooting.md b/website/content/en/v0.32/troubleshooting.md index cb51859789d9..d0c79437f962 100644 --- a/website/content/en/v0.32/troubleshooting.md +++ b/website/content/en/v0.32/troubleshooting.md @@ -61,7 +61,7 @@ Node role names for Karpenter are created in the form `KarpenterNodeRole-${Clust If a long cluster name causes the Karpenter node role name to exceed 64 characters, creating that object will fail. Keep in mind that `KarpenterNodeRole-` is just a recommendation from the getting started guide. -Instead using of the eksctl role, you can shorten the name to anything you like, as long as it has the right permissions. +Instead of using the eksctl role, you can shorten the name to anything you like, as long as it has the right permissions. ### Unknown field in Provisioner spec @@ -111,8 +111,9 @@ kubectl label crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodecl - In the case of `annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "karpenter"` run: ```shell +KARPENTER_NAMESPACE=karpenter kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite -kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace="${KARPENTER_NAMESPACE}" --overwrite ``` ## Uninstallation @@ -318,6 +319,11 @@ By default, the number of pods on a node is limited by both the number of networ If the max-pods (configured through your Provisioner [`kubeletConfiguration`]({{}})) is greater than the number of supported IPs for a given instance type, the CNI will fail to assign an IP to the pod and your pod will be left in a `ContainerCreating` state. +If you've enabled [Security Groups per Pod](https://aws.github.io/aws-eks-best-practices/networking/sgpp/), one of the instance's ENIs is reserved as the trunk interface and uses branch interfaces off of that trunk interface to assign different security groups. +If you do not have any `SecurityGroupPolicies` configured for your pods, they will be unable to utilize branch interfaces attached to the trunk interface, and IPs will only be available from the non-trunk ENIs. +This effectively reduces the max-pods value by the number of IPs that would have been available from the trunk ENI. +Note that Karpenter is not aware if [Security Groups per Pod](https://aws.github.io/aws-eks-best-practices/networking/sgpp/) is enabled, and will continue to compute max-pods assuming all ENIs on the instance can be utilized. + ##### Solutions To avoid this discrepancy between `maxPods` and the supported pod density of the EC2 instance based on ENIs and allocatable IPs, you can perform one of the following actions on your cluster: @@ -635,7 +641,7 @@ To correct the problem if it occurs, you can use the approach that AWS EBS uses, "AWS": "arn:aws:iam::${AWS_ACCOUNT_ID}:root" }, "Action": [ - "kms:Describe", + "kms:Describe*", "kms:Get*", "kms:List*", "kms:RevokeGrant" @@ -651,7 +657,7 @@ This typically occurs when the node has not been considered fully initialized fo ### Log message of `inflight check failed for node, Expected resource "vpc.amazonaws.com/pod-eni" didn't register on the node` is reported -This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. If you've enabled Pod ENI for Karpenter nodes via the `aws.enablePodENI` setting, you will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. +This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. You will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. ### AWS Node Termination Handler (NTH) interactions Karpenter [doesn't currently support draining and terminating on spot rebalance recommendations]({{< ref "concepts/disruption#interruption" >}}). Users who want support for both drain and terminate on spot interruption as well as drain and termination on spot rebalance recommendations may install Node Termination Handler (NTH) on their clusters to support this behavior. diff --git a/website/content/en/v0.32/upgrading/compatibility.md b/website/content/en/v0.32/upgrading/compatibility.md index 836df2552a15..bbf6603136e3 100644 --- a/website/content/en/v0.32/upgrading/compatibility.md +++ b/website/content/en/v0.32/upgrading/compatibility.md @@ -71,14 +71,13 @@ Karpenter offers three types of releases. This section explains the purpose of e ### Stable Releases -Stable releases are the most reliable releases that are released with weekly cadence. Stable releases are our only recommended versions for production environments. -Sometimes we skip a stable release because we find instability or problems that need to be fixed before having a stable release. -Stable releases are tagged with Semantic Versioning. For example `v0.13.0`. +Stable releases are the only recommended versions for production environments. Stable releases are tagged with a semantic version (e.g. `0.35.0`). Note that stable releases prior to `0.35.0` are prefixed with a `v` (e.g. `v0.34.0`). ### Release Candidates -We consider having release candidates for major and important minor versions. Our release candidates are tagged like `vx.y.z-rc.0`, `vx.y.z-rc.1`. The release candidate will then graduate to `vx.y.z` as a normal stable release. +We consider having release candidates for major and important minor versions. Our release candidates are tagged like `x.y.z-rc.0`, `x.y.z-rc.1`. The release candidate will then graduate to `x.y.z` as a stable release. By adopting this practice we allow our users who are early adopters to test out new releases before they are available to the wider community, thereby providing us with early feedback resulting in more stable releases. +Note that, like the stable releases, release candidates prior to `0.35.0` are prefixed with a `v`. ### Snapshot Releases diff --git a/website/content/en/v0.32/upgrading/upgrade-guide.md b/website/content/en/v0.32/upgrading/upgrade-guide.md index 5deb0741f0b2..dc45fc1a1df8 100644 --- a/website/content/en/v0.32/upgrading/upgrade-guide.md +++ b/website/content/en/v0.32/upgrading/upgrade-guide.md @@ -28,12 +28,12 @@ If you get the error `invalid ownership metadata; label validation error:` while In general, you can reapply the CRDs in the `crds` directory of the Karpenter helm chart: ```shell -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_provisioners.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_machines.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_nodepools.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_nodeclaims.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.sh_provisioners.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.sh_machines.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.sh_nodepools.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml ``` ### Upgrading to v0.32.0+ @@ -43,9 +43,10 @@ Karpenter v0.32.0 introduces v1beta1 APIs, including _significant_ changes to th This version includes **dual support** for both alpha and beta APIs to ensure that you can slowly migrate your existing Provisioner, AWSNodeTemplate, and Machine alpha APIs to the newer NodePool, EC2NodeClass, and NodeClaim beta APIs. -Note that if you are rolling back after upgrading to v0.32.0, note that v0.31.4 is the only version that supports handling rollback after you have deployed the v1beta1 APIs to your cluster. +Note that if you are rolling back after upgrading to v0.32.0, note that __only__ versions v0.31.4+ support handling rollback after you have deployed the v1beta1 APIs to your cluster. {{% /alert %}} +* Karpenter now uses `settings.InterruptionQueue` instead of `settings.aws.InterruptionQueueName` in its helm chart. The CLI argument also changed to `--interruption-queue`. * Karpenter now serves the webhook prometheus metrics server on port `8001`. If this port is already in-use on the pod or you are running in `hostNetworking` mode, you may need to change this port value. You can configure this port value through the `WEBHOOK_METRICS_PORT` environment variable or the `webhook.metrics.port` value if installing via Helm. * Karpenter now exposes the ability to disable webhooks through the `webhook.enabled=false` value. This value will disable the webhook server and will prevent any permissions, mutating or validating webhook configurations from being deployed to the cluster. * Karpenter now moves all logging configuration for the Zap logger into the `logConfig` values block. Configuring Karpenter logging with this mechanism _is_ deprecated and will be dropped at v1. Karpenter now only surfaces logLevel through the `logLevel` helm value. If you need more advanced configuration due to log parsing constraints, we recommend configuring your log parser to handle Karpenter's Zap JSON logging. diff --git a/website/content/en/v0.32/upgrading/v1beta1-migration.md b/website/content/en/v0.32/upgrading/v1beta1-migration.md index 84e5fb5042ad..d048935b6547 100644 --- a/website/content/en/v0.32/upgrading/v1beta1-migration.md +++ b/website/content/en/v0.32/upgrading/v1beta1-migration.md @@ -38,7 +38,7 @@ This procedure assumes you are running the Karpenter controller on cluster and w ``` {{% alert title="Warning" color="warning" %}} - v0.31.2 introduces minor changes to Karpenter so that rollback from v0.32.0 is supported. If you are coming from some other patch version of minor version v0.31.x, note that v0.31.4 is the _only_ patch version that supports rollback for v1beta1. + v0.31.2 introduces minor changes to Karpenter so that rollback from v0.32.0 is supported. If you are coming from some other patch version of minor version v0.31.x, note that __only__ versions v0.31.4+ support rollback for v1beta1. {{% /alert %}} 2. Review for breaking changes: If you are already running Karpenter v0.31.x, you can skip this step. If you are running an earlier Karpenter version, you need to review the upgrade notes for each minor release. @@ -47,7 +47,7 @@ This procedure assumes you are running the Karpenter controller on cluster and w ```bash export KARPENTER_NAMESPACE=karpenter - export KARPENTER_VERSION=v0.32.9 + export KARPENTER_VERSION=v0.32.10 export AWS_PARTITION="aws" # if you are not using standard partitions, you may need to configure to aws-cn / aws-us-gov export CLUSTER_NAME="${USER}-karpenter-demo" export AWS_REGION="us-west-2" @@ -60,7 +60,7 @@ This procedure assumes you are running the Karpenter controller on cluster and w ```bash TEMPOUT=$(mktemp) - curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/website/content/en/preview/upgrading/v1beta1-controller-policy.json > ${TEMPOUT} + curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/website/content/en/preview/upgrading/v1beta1-controller-policy.json > ${TEMPOUT} AWS_REGION=${AWS_REGION:=$AWS_DEFAULT_REGION} # use the default region if AWS_REGION isn't defined POLICY_DOCUMENT=$(envsubst < ${TEMPOUT}) @@ -71,15 +71,15 @@ This procedure assumes you are running the Karpenter controller on cluster and w aws iam attach-role-policy --role-name "${ROLE_NAME}" --policy-arn "${POLICY_ARN}" ``` -5. Apply the v0.32.9 Custom Resource Definitions (CRDs): +5. Apply the v0.32.10 Custom Resource Definitions (CRDs): ```bash - kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_provisioners.yaml - kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_machines.yaml - kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml - kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_nodepools.yaml - kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_nodeclaims.yaml - kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.sh_provisioners.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.sh_machines.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.sh_nodepools.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.sh_nodeclaims.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml ``` 6. Upgrade Karpenter to the new version: @@ -758,7 +758,7 @@ Karpenter v1beta1 introduces changes to some common labels, annotations, and sta v1beta1 introduces changes to the IAM permissions assigned to the Karpenter controller policy used when deploying Karpenter to your cluster with [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) or [EKS Pod Identity](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html). -You can see a full example of the v1beta1 required controller permissions by viewing the [v1beta1 Controller Policy](https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/website/content/en/preview/upgrading/v1beta1-controller-policy.json). +You can see a full example of the v1beta1 required controller permissions by viewing the [v1beta1 Controller Policy](https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/website/content/en/preview/upgrading/v1beta1-controller-policy.json). Additionally, read more detail about the full set of permissions assigned to the Karpenter controller policy in the [CloudFormation Reference Guide]({{< ref "../reference/cloudformation" >}}). diff --git a/website/content/en/v0.34/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json deleted file mode 100644 index d474d01f4e16..000000000000 --- a/website/content/en/v0.34/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json +++ /dev/null @@ -1,1440 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 2, - "id": 6, - "links": [], - "liveNow": true, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(action, cluster) (karpenter_deprovisioning_actions_performed)", - "format": "time_series", - "instant": false, - "legendFormat": "{{cluster}}: {{action}}", - "range": true, - "refId": "A" - } - ], - "title": "Deprovisioning Actions Performed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "sum by(cluster) (karpenter_nodes_created)", - "format": "time_series", - "legendFormat": "{{cluster}}", - "range": true, - "refId": "A" - } - ], - "title": "Nodes Created", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 15, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "sum by(cluster) (karpenter_nodes_terminated)", - "format": "time_series", - "legendFormat": "{{cluster}}", - "range": true, - "refId": "A" - } - ], - "title": "Nodes Terminated", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by(phase)(karpenter_pods_state)", - "legendFormat": "{{label_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Pod Phase", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", - "legendFormat": "{{label_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Pod Distribution: $distribution_filter", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-RdYlGr" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*Utilization$" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 1 - }, - { - "id": "unit", - "value": "percentunit" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Provisioned" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 18, - "x": 0, - "y": 29 - }, - "id": 10, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "9.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "karpenter_nodepool_usage{resource_type=\"cpu\"} / karpenter_nodepool_limit{resource_type=\"cpu\"}", - "format": "table", - "instant": true, - "legendFormat": "CPU Limit Utilization", - "range": false, - "refId": "CPU Limit Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Node Count" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "karpenter_nodepool_usage{resource_type=\"memory\"} / karpenter_nodepool_limit{resource_type=\"memory\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Memory Limit Utilization", - "range": false, - "refId": "Memory Limit Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "CPU Capacity" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Memory Capacity" - } - ], - "title": "Nodepool Summary", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "nodepool" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "__name__": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "job": true, - "job 1": true, - "job 2": true, - "resource_type": true, - "resource_type 1": true, - "resource_type 2": true - }, - "indexByName": { - "Time 1": 6, - "Time 2": 7, - "Time 3": 11, - "Time 4": 15, - "Time 5": 16, - "Value #CPU Capacity": 2, - "Value #CPU Limit Utilization": 3, - "Value #Memory Capacity": 4, - "Value #Memory Limit Utilization": 5, - "Value #Node Count": 1, - "instance 1": 8, - "instance 2": 12, - "job 1": 9, - "job 2": 13, - "nodepool": 0, - "resource_type 1": 10, - "resource_type 2": 14 - }, - "renameByName": { - "Time 1": "", - "Value": "CPU Utilization", - "Value #CPU Capacity": "CPU Provisioned", - "Value #CPU Limit Utilization": "CPU Limit Utilization", - "Value #CPU Utilization": "CPU Limit Utilization", - "Value #Memory Capacity": "Memory Provisioned", - "Value #Memory Limit Utilization": "Memory Limit Utilization", - "Value #Memory Utilization": "Memory Utilization", - "Value #Node Count": "Node Count", - "instance": "", - "instance 1": "", - "job": "", - "nodepool": "Nodepool" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 18, - "y": 29 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", - "legendFormat": "Percentage", - "range": true, - "refId": "A" - } - ], - "title": "Spot Node Percentage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-RdYlGr" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "node_name" - }, - "properties": [ - { - "id": "custom.width", - "value": 333 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Utilization" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "thresholds", - "value": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 75 - } - ] - } - }, - { - "id": "max", - "value": 1 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "unit", - "value": "s" - }, - { - "id": "decimals", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 4, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Uptime" - } - ] - }, - "pluginVersion": "9.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "CPU Utilization", - "range": false, - "refId": "CPU Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Memory Utilization", - "range": false, - "refId": "Memory Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Memory Utilization", - "range": false, - "refId": "Pod Count" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\n sum by (node)(node_time_seconds) - sum by (node)(node_boot_time_seconds),\n \"node_name\", \"$1\", \"node\", \"(.+)\"\n)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Uptime", - "range": false, - "refId": "Uptime" - } - ], - "title": "Node Summary", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "node_name" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Value": false, - "Value #Pod Count": false, - "__name__": true, - "arch": true, - "arch 1": true, - "arch 2": true, - "arch 3": true, - "capacity_type 2": true, - "capacity_type 3": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "instance 3": true, - "instance_category 1": true, - "instance_category 2": true, - "instance_category 3": true, - "instance_cpu": true, - "instance_cpu 1": true, - "instance_cpu 2": true, - "instance_cpu 3": true, - "instance_family": true, - "instance_family 1": true, - "instance_family 2": true, - "instance_family 3": true, - "instance_generation 1": true, - "instance_generation 2": true, - "instance_generation 3": true, - "instance_gpu_count": true, - "instance_gpu_count 1": true, - "instance_gpu_count 2": true, - "instance_gpu_count 3": true, - "instance_gpu_manufacturer": true, - "instance_gpu_manufacturer 1": true, - "instance_gpu_manufacturer 2": true, - "instance_gpu_manufacturer 3": true, - "instance_gpu_memory": true, - "instance_gpu_memory 1": true, - "instance_gpu_memory 2": true, - "instance_gpu_memory 3": true, - "instance_gpu_name": true, - "instance_gpu_name 1": true, - "instance_gpu_name 2": true, - "instance_gpu_name 3": true, - "instance_hypervisor": true, - "instance_hypervisor 1": true, - "instance_hypervisor 2": true, - "instance_hypervisor 3": true, - "instance_local_nvme 1": true, - "instance_local_nvme 2": true, - "instance_local_nvme 3": true, - "instance_memory": true, - "instance_memory 1": true, - "instance_memory 2": true, - "instance_memory 3": true, - "instance_pods": true, - "instance_pods 1": true, - "instance_pods 2": true, - "instance_pods 3": true, - "instance_size": true, - "instance_size 1": true, - "instance_size 2": true, - "instance_size 3": true, - "instance_type 1": false, - "instance_type 2": true, - "instance_type 3": true, - "job": true, - "job 1": true, - "job 2": true, - "job 3": true, - "node": true, - "os": true, - "os 1": true, - "os 2": true, - "os 3": true, - "nodepool 1": false, - "nodepool 2": true, - "nodepool 3": true, - "resource_type": true, - "resource_type 1": true, - "resource_type 2": true, - "resource_type 3": true, - "zone 1": false, - "zone 2": true, - "zone 3": true - }, - "indexByName": { - "Time 1": 1, - "Time 2": 25, - "Time 3": 45, - "Time 4": 65, - "Value #CPU Utilization": 10, - "Value #Memory Utilization": 11, - "Value #Pod Count": 9, - "Value #Uptime": 8, - "arch 1": 5, - "arch 2": 26, - "arch 3": 46, - "capacity_type 1": 6, - "capacity_type 2": 27, - "capacity_type 3": 47, - "instance 1": 4, - "instance 2": 28, - "instance 3": 48, - "instance_cpu 1": 12, - "instance_cpu 2": 29, - "instance_cpu 3": 49, - "instance_family 1": 13, - "instance_family 2": 30, - "instance_family 3": 50, - "instance_gpu_count 1": 14, - "instance_gpu_count 2": 31, - "instance_gpu_count 3": 51, - "instance_gpu_manufacturer 1": 15, - "instance_gpu_manufacturer 2": 32, - "instance_gpu_manufacturer 3": 52, - "instance_gpu_memory 1": 16, - "instance_gpu_memory 2": 33, - "instance_gpu_memory 3": 53, - "instance_gpu_name 1": 17, - "instance_gpu_name 2": 34, - "instance_gpu_name 3": 54, - "instance_hypervisor 1": 18, - "instance_hypervisor 2": 35, - "instance_hypervisor 3": 55, - "instance_memory 1": 19, - "instance_memory 2": 36, - "instance_memory 3": 56, - "instance_pods 1": 20, - "instance_pods 2": 37, - "instance_pods 3": 57, - "instance_size 1": 21, - "instance_size 2": 38, - "instance_size 3": 58, - "instance_type 1": 3, - "instance_type 2": 39, - "instance_type 3": 59, - "job 1": 22, - "job 2": 40, - "job 3": 60, - "node": 66, - "node_name": 0, - "os 1": 23, - "os 2": 41, - "os 3": 61, - "nodepool 1": 2, - "nodepool 2": 42, - "nodepool 3": 62, - "resource_type 1": 24, - "resource_type 2": 43, - "resource_type 3": 63, - "zone 1": 7, - "zone 2": 44, - "zone 3": 64 - }, - "renameByName": { - "Time": "", - "Time 1": "", - "Value": "CPU Utilization", - "Value #Allocatable": "", - "Value #CPU Utilization": "CPU Utilization", - "Value #Memory Utilization": "Memory Utilization", - "Value #Pod CPU": "", - "Value #Pod Count": "Pods", - "Value #Uptime": "Uptime", - "arch": "Architecture", - "arch 1": "Arch", - "capacity_type": "Capacity Type", - "capacity_type 1": "Capacity Type", - "instance 1": "Instance", - "instance_cpu 1": "vCPU", - "instance_type": "Instance Type", - "instance_type 1": "Instance Type", - "node_name": "Node Name", - "nodepool 1": "Nodepool", - "zone 1": "Zone" - } - } - } - ], - "type": "table" - } - ], - "refresh": false, - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "includeAll": false, - "label": "Data Source", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, nodepool)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "nodepool", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, nodepool)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, zone)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "zone", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, zone)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, arch)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "arch", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, arch)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, capacity_type)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "capacity_type", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, capacity_type)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, instance_type)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "instance_type", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, instance_type)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": "nodepool", - "value": "nodepool" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "distribution_filter", - "options": [ - { - "selected": false, - "text": "arch", - "value": "arch" - }, - { - "selected": false, - "text": "capacity_type", - "value": "capacity_type" - }, - { - "selected": false, - "text": "instance_type", - "value": "instance_type" - }, - { - "selected": false, - "text": "namespace", - "value": "namespace" - }, - { - "selected": false, - "text": "node", - "value": "node" - }, - { - "selected": true, - "text": "nodepool", - "value": "nodepool" - }, - { - "selected": false, - "text": "zone", - "value": "zone" - } - ], - "query": "arch,capacity_type,instance_type,namespace,node,nodepool,zone", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Karpenter Capacity", - "uid": "ta8I9Q67z", - "version": 4, - "weekStart": "" -} diff --git a/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step16-delete-node.sh b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step16-delete-node.sh deleted file mode 100755 index 9d431160dda0..000000000000 --- a/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step16-delete-node.sh +++ /dev/null @@ -1 +0,0 @@ -kubectl delete node $NODE_NAME diff --git a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step03-node-policies.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step03-node-policies.sh deleted file mode 100644 index d57f79039d04..000000000000 --- a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step03-node-policies.sh +++ /dev/null @@ -1,11 +0,0 @@ -aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKSWorkerNodePolicy - -aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKS_CNI_Policy - -aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn arn:${AWS_PARTITION}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly - -aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn arn:${AWS_PARTITION}:iam::aws:policy/AmazonSSMManagedInstanceCore diff --git a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh deleted file mode 100644 index de972ea2bddd..000000000000 --- a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh +++ /dev/null @@ -1,6 +0,0 @@ -for NODEGROUP in $(aws eks list-nodegroups --cluster-name ${CLUSTER_NAME} \ - --query 'nodegroups' --output text); do aws ec2 create-tags \ - --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ - --resources $(aws eks describe-nodegroup --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name $NODEGROUP --query 'nodegroup.subnets' --output text ) -done diff --git a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh deleted file mode 100644 index 397e40904cee..000000000000 --- a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh +++ /dev/null @@ -1,22 +0,0 @@ -NODEGROUP=$(aws eks list-nodegroups --cluster-name ${CLUSTER_NAME} \ - --query 'nodegroups[0]' --output text) - -LAUNCH_TEMPLATE=$(aws eks describe-nodegroup --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name ${NODEGROUP} --query 'nodegroup.launchTemplate.{id:id,version:version}' \ - --output text | tr -s "\t" ",") - -# If your EKS setup is configured to use only Cluster security group, then please execute - - -SECURITY_GROUPS=$(aws eks describe-cluster \ - --name ${CLUSTER_NAME} --query "cluster.resourcesVpcConfig.clusterSecurityGroupId" --output text) - -# If your setup uses the security groups in the Launch template of a managed node group, then : - -SECURITY_GROUPS=$(aws ec2 describe-launch-template-versions \ - --launch-template-id ${LAUNCH_TEMPLATE%,*} --versions ${LAUNCH_TEMPLATE#*,} \ - --query 'LaunchTemplateVersions[0].LaunchTemplateData.[NetworkInterfaces[0].Groups||SecurityGroupIds]' \ - --output text) - -aws ec2 create-tags \ - --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ - --resources ${SECURITY_GROUPS} diff --git a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step09-deploy.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step09-deploy.sh deleted file mode 100644 index 51714d78f6dd..000000000000 --- a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step09-deploy.sh +++ /dev/null @@ -1,8 +0,0 @@ -kubectl create namespace "${KARPENTER_NAMESPACE}" || true -kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodepools.yaml -kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml -kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodeclaims.yaml -kubectl apply -f karpenter.yaml diff --git a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh deleted file mode 100644 index d88a6d5c7236..000000000000 --- a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh +++ /dev/null @@ -1,5 +0,0 @@ -for NODEGROUP in $(aws eks list-nodegroups --cluster-name ${CLUSTER_NAME} \ - --query 'nodegroups' --output text); do aws eks update-nodegroup-config --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name ${NODEGROUP} \ - --scaling-config "minSize=1,maxSize=1,desiredSize=1" -done diff --git a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh deleted file mode 100644 index 51ad964c28a7..000000000000 --- a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh +++ /dev/null @@ -1,3 +0,0 @@ -aws eks update-nodegroup-config --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name ${NODEGROUP} \ - --scaling-config "minSize=2,maxSize=2,desiredSize=2" diff --git a/website/content/en/v0.35/concepts/_index.md b/website/content/en/v0.35/concepts/_index.md index 67d573135369..e7f8ce4cb500 100755 --- a/website/content/en/v0.35/concepts/_index.md +++ b/website/content/en/v0.35/concepts/_index.md @@ -29,7 +29,7 @@ Once privileges are in place, Karpenter is deployed with a Helm chart. ### Configuring NodePools -Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. To configure Karpenter, you create [NodePools]({{}}) that define how Karpenter manages unschedulable pods and configures nodes. You will also define behaviors for your NodePools, capturing details like how Karpenter handles disruption of nodes and setting limits and weights for each NodePool +Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. To configure Karpenter, you create [NodePools]({{}}) that define how Karpenter manages unschedulable pods and configures nodes. You will also define behaviors for your NodePools, capturing details like how Karpenter handles disruption of nodes and setting limits and weights for each NodePool. Here are some things to know about Karpenter's NodePools: diff --git a/website/content/en/v0.35/concepts/disruption.md b/website/content/en/v0.35/concepts/disruption.md index 48e0b479c5f3..ccefdd91db0a 100644 --- a/website/content/en/v0.35/concepts/disruption.md +++ b/website/content/en/v0.35/concepts/disruption.md @@ -186,7 +186,7 @@ If you require handling for Spot Rebalance Recommendations, you can use the [AWS Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). -To enable interruption handling, configure the `--interruption-queue-name` CLI argument with the name of the interruption queue provisioned to handle interruption events. +To enable interruption handling, configure the `--interruption-queue` CLI argument with the name of the interruption queue provisioned to handle interruption events. ## Controls @@ -223,7 +223,7 @@ spec: #### Schedule Schedule is a cronjob schedule. Generally, the cron syntax is five space-delimited values with options below, with additional special macros like `@yearly`, `@monthly`, `@weekly`, `@daily`, `@hourly`. -Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#writing-a-cronjob-spec) for more information on how to follow the cron syntax. +Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#writing-a-cronjob-spec) for more information on how to follow the cron syntax. Timezones are not currently supported. Schedules are always in UTC. ```bash # ┌───────────── minute (0 - 59) @@ -237,10 +237,6 @@ Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/worklo # * * * * * ``` -{{% alert title="Note" color="primary" %}} -Timezones are not supported. Most images default to UTC, but it is best practice to ensure this is the case when considering how to define your budgets. -{{% /alert %}} - #### Duration Duration allows compound durations with minutes and hours values such as `10h5m` or `30m` or `160h`. Since cron syntax does not accept denominations smaller than minutes, users can only define minutes or hours. @@ -291,7 +287,7 @@ metadata: #### Example: Disable Disruption on a NodePool -NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in disruption actions. +To disable disruption for all nodes launched by a NodePool, you can configure its `.spec.disruption.budgets`. Setting a budget of zero nodes will prevent any of those nodes from being considered for voluntary disruption. ```yaml apiVersion: karpenter.sh/v1beta1 @@ -299,8 +295,7 @@ kind: NodePool metadata: name: default spec: - template: - metadata: - annotations: # will be applied to all nodes - karpenter.sh/do-not-disrupt: "true" + disruption: + budgets: + - nodes: "0" ``` diff --git a/website/content/en/v0.35/concepts/nodeclasses.md b/website/content/en/v0.35/concepts/nodeclasses.md index d426f579e7e6..5f5324027f48 100644 --- a/website/content/en/v0.35/concepts/nodeclasses.md +++ b/website/content/en/v0.35/concepts/nodeclasses.md @@ -160,7 +160,7 @@ status: # Generated instance profile name from "role" instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" ``` -Refer to the [NodePool docs]({{}}) for settings applicable to all providers. To explore various `EC2NodeClass` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). +Refer to the [NodePool docs]({{}}) for settings applicable to all providers. To explore various `EC2NodeClass` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/v0.35.0/examples/v1beta1/). ## spec.amiFamily @@ -431,6 +431,10 @@ spec: AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). **When you specify `amiSelectorTerms`, you fully override the default AMIs that are selected on by your EC2NodeClass [`amiFamily`]({{< ref "#specamifamily" >}}).** +{{% alert title="Note" color="primary" %}} +[`amiFamily`]({{< ref "#specamifamily" >}}) determines the bootstrapping mode, while `amiSelectorTerms` specifies specific AMIs to be used. Therefore, you need to ensure consistency between [`amiFamily`]({{< ref "#specamifamily" >}}) and `amiSelectorTerms` to avoid conflicts during bootstrapping. +{{% /alert %}} + This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. ```yaml @@ -745,7 +749,7 @@ spec: chown -R ec2-user ~ec2-user/.ssh ``` -For more examples on configuring fields for different AMI families, see the [examples here](https://github.com/aws/karpenter/blob/main/examples/v1beta1). +For more examples on configuring fields for different AMI families, see the [examples here](https://github.com/aws/karpenter/blob/v0.35.0/examples/v1beta1/). Karpenter will merge the userData you specify with the default userData for that AMIFamily. See the [AMIFamily]({{< ref "#specamifamily" >}}) section for more details on these defaults. View the sections below to understand the different merge strategies for each AMIFamily. diff --git a/website/content/en/v0.35/concepts/nodepools.md b/website/content/en/v0.35/concepts/nodepools.md index d679bb5f4373..03cc87370502 100644 --- a/website/content/en/v0.35/concepts/nodepools.md +++ b/website/content/en/v0.35/concepts/nodepools.md @@ -22,7 +22,7 @@ Here are things you should know about NodePools: * If Karpenter encounters a startup taint in the NodePool it will be applied to nodes that are provisioned, but pods do not need to tolerate the taint. Karpenter assumes that the taint is temporary and some other system will remove the taint. * It is recommended to create NodePools that are mutually exclusive. So no Pod should match multiple NodePools. If multiple NodePools are matched, Karpenter will use the NodePool with the highest [weight](#specweight). -For some example `NodePool` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). +For some example `NodePool` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/v0.35.0/examples/v1beta1/). ```yaml apiVersion: karpenter.sh/v1beta1 @@ -157,7 +157,7 @@ spec: duration: 8h nodes: "0" - # Resource limits constrain the total size of the cluster. + # Resource limits constrain the total size of the pool. # Limits prevent Karpenter from creating new instances once the limit is exceeded. limits: cpu: "1000" @@ -233,6 +233,10 @@ Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand i Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. +{{% alert title="Note" color="primary" %}} +There is currently a limit of 30 on the total number of requirements on both the NodePool and the NodeClaim. It's important to note that `spec.template.metadata.labels` are also propagated as requirements on the NodeClaim when it's created, meaning that you can't have more than 30 requirements and labels combined set on your NodePool. +{{% /alert %}} + ### Min Values Along with the combination of [key,operator,values] in the requirements, Karpenter also supports `minValues` in the NodePool requirements block, allowing the scheduler to be aware of user-specified flexibility minimums while scheduling pods to a cluster. If Karpenter cannot meet this minimum flexibility for each key when scheduling a pod, it will fail the scheduling loop for that NodePool, either falling back to another NodePool which meets the pod requirements or failing scheduling the pod altogether. diff --git a/website/content/en/v0.35/concepts/scheduling.md b/website/content/en/v0.35/concepts/scheduling.md index ef12a29cc347..0674870f5b07 100755 --- a/website/content/en/v0.35/concepts/scheduling.md +++ b/website/content/en/v0.35/concepts/scheduling.md @@ -104,8 +104,6 @@ Refer to general [Kubernetes GPU](https://kubernetes.io/docs/tasks/manage-gpus/s You must enable Pod ENI support in the AWS VPC CNI Plugin before enabling Pod ENI support in Karpenter. Please refer to the [Security Groups for Pods documentation](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) for instructions. {{% /alert %}} -Now that Pod ENI support is enabled in the AWS VPC CNI Plugin, you can enable Pod ENI support in Karpenter by setting the `settings.aws.enablePodENI` Helm chart value to `true`. - Here is an example of a pod-eni resource defined in a deployment manifest: ``` spec: @@ -174,6 +172,9 @@ requirements: - key: user.defined.label/type operator: Exists ``` +{{% alert title="Note" color="primary" %}} +There is currently a limit of 30 on the total number of requirements on both the NodePool and the NodeClaim. It's important to note that `spec.template.metadata.labels` are also propagated as requirements on the NodeClaim when it's created, meaning that you can't have more than 30 requirements and labels combined set on your NodePool. +{{% /alert %}} #### Node selectors @@ -281,7 +282,7 @@ spec: - p3 taints: - key: nvidia.com/gpu - value: true + value: "true" effect: "NoSchedule" ``` @@ -625,19 +626,73 @@ If using Gt/Lt operators, make sure to use values under the actual label values The `Exists` operator can be used on a NodePool to provide workload segregation across nodes. ```yaml -... -requirements: -- key: company.com/team - operator: Exists +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +spec: + template: + spec: + requirements: + - key: company.com/team + operator: Exists ... ``` -With the requirement on the NodePool, workloads can optionally specify a custom value as a required node affinity or node selector. Karpenter will then label the nodes it launches for these pods which prevents `kube-scheduler` from scheduling conflicting pods to those nodes. This provides a way to more dynamically isolate workloads without requiring a unique NodePool for each workload subset. +With this requirement on the NodePool, workloads can specify the same key (e.g. `company.com/team`) with custom values (e.g. `team-a`, `team-b`, etc.) as a required `nodeAffinity` or `nodeSelector`. Karpenter will then apply the key/value pair to nodes it launches dynamically based on the pod's node requirements. + +If each set of pods that can schedule with this NodePool specifies this key in its `nodeAffinity` or `nodeSelector`, you can isolate pods onto different nodes based on their values. This provides a way to more dynamically isolate workloads without requiring a unique NodePool for each workload subset. + +For example, providing the following `nodeSelectors` would isolate the pods for each of these deployments on different nodes. + +#### Team A Deployment ```yaml -nodeSelector: - company.com/team: team-a +apiVersion: v1 +kind: Deployment +metadata: + name: team-a-deployment +spec: + replicas: 5 + template: + spec: + nodeSelector: + company.com/team: team-a ``` + +#### Team A Node + +```yaml +apiVersion: v1 +kind: Node +metadata: + labels: + company.com/team: team-a +``` + +#### Team B Deployment + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: team-b-deployment +spec: + replicas: 5 + template: + spec: + nodeSelector: + company.com/team: team-b +``` + +#### Team B Node + +```yaml +apiVersion: v1 +kind: Node +metadata: + labels: + company.com/team: team-b +``` + {{% alert title="Note" color="primary" %}} If a workload matches the NodePool but doesn't specify a label, Karpenter will generate a random label for the node. {{% /alert %}} diff --git a/website/content/en/v0.35/faq.md b/website/content/en/v0.35/faq.md index a825e34056f6..e55b519a50e1 100644 --- a/website/content/en/v0.35/faq.md +++ b/website/content/en/v0.35/faq.md @@ -14,10 +14,10 @@ See [Configuring NodePools]({{< ref "./concepts/#configuring-nodepools" >}}) for AWS is the first cloud provider supported by Karpenter, although it is designed to be used with other cloud providers as well. ### Can I write my own cloud provider for Karpenter? -Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.35.4/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. +Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.35.5/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. ### What operating system nodes does Karpenter deploy? -Karpenter uses the OS defined by the [AMI Family in your EC2NodeClass]({{< ref "./concepts/nodeclasses#specamifamily" >}}). +Karpenter uses the OS defined by the [AMI Family in your EC2NodeClass]({{< ref "./concepts/nodeclasses#specamifamily" >}}). ### Can I provide my own custom operating system images? Karpenter has multiple mechanisms for configuring the [operating system]({{< ref "./concepts/nodeclasses/#specamiselectorterms" >}}) for your nodes. @@ -26,7 +26,7 @@ Karpenter has multiple mechanisms for configuring the [operating system]({{< ref Karpenter is flexible to multi-architecture configurations using [well known labels]({{< ref "./concepts/scheduling/#supported-labels">}}). ### What RBAC access is required? -All the required RBAC rules can be found in the Helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/role.yaml) files for details. +All the required RBAC rules can be found in the Helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.35.5/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.35.5/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.35.5/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.35.5/charts/karpenter/templates/role.yaml) files for details. ### Can I run Karpenter outside of a Kubernetes cluster? Yes, as long as the controller has network and IAM/RBAC access to the Kubernetes API and your provider API. @@ -211,15 +211,15 @@ For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgra ### How do I upgrade an EKS Cluster with Karpenter? -When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift is enabled by default starting `0.33.0`. - {{% alert title="Note" color="primary" %}} -Karpenter's default [EC2NodeClass `amiFamily` configuration]({{}}) uses the latest EKS Optimized AL2 AMI for the same major and minor version as the EKS cluster's control plane, meaning that an upgrade of the control plane will cause Karpenter to auto-discover the new AMIs for that version. +Karpenter recommends that you always validate AMIs in your lower environments before using them in production environments. Read [Managing AMIs]({{}}) to understand best practices about upgrading your AMIs. -If using a custom AMI, you will need to trigger the rollout of this new worker node image through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. +If using a custom AMI, you will need to trigger the rollout of new worker node images through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. {{% /alert %}} -Start by [upgrading the EKS Cluster control plane](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). After the EKS Cluster upgrade completes, Karpenter's Drift feature will detect that the Karpenter-provisioned nodes are using EKS Optimized AMIs for the previous cluster version, and [automatically cordon, drain, and replace those nodes]({{}}). To support pods moving to new nodes, follow Kubernetes best practices by setting appropriate pod [Resource Quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/), and using [Pod Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (PDB). Karpenter's Drift feature will spin up replacement nodes based on the pod resource requests, and will respect the PDBs when deprovisioning nodes. +Karpenter's default behavior will upgrade your nodes when you've upgraded your Amazon EKS Cluster. Karpenter will [drift]({{}}) nodes to stay in-sync with the EKS control plane version. Drift is enabled by default starting in `v0.33`. This means that as soon as your cluster is upgraded, Karpenter will auto-discover the new AMIs for that version. + +Start by [upgrading the EKS Cluster control plane](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). After the EKS Cluster upgrade completes, Karpenter will Drift and disrupt the Karpenter-provisioned nodes using EKS Optimized AMIs for the previous cluster version by first spinning up replacement nodes. Karpenter respects [Pod Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (PDB), and automatically [replaces, cordons, and drains those nodes]({{}}). To best support pods moving to new nodes, follow Kubernetes best practices by setting appropriate pod [Resource Quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/) and using PDBs. ## Interruption Handling @@ -231,7 +231,7 @@ Karpenter's native interruption handling offers two main benefits over the stand 1. You don't have to manage and maintain a separate component to exclusively handle interruption events. 2. Karpenter's native interruption handling coordinates with other deprovisioning so that consolidation, expiration, etc. can be aware of interruption events and vice-versa. -### Why am I receiving QueueNotFound errors when I set `--interruption-queue-name`? +### Why am I receiving QueueNotFound errors when I set `--interruption-queue`? Karpenter requires a queue to exist that receives event messages from EC2 and health services in order to handle interruption messages properly for nodes. Details on the types of events that Karpenter handles can be found in the [Interruption Handling Docs]({{< ref "./concepts/disruption/#interruption" >}}). diff --git a/website/content/en/v0.35/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/_index.md index 6cb0c1692cee..41a165b7e414 100644 --- a/website/content/en/v0.35/getting-started/getting-started-with-karpenter/_index.md +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/_index.md @@ -45,7 +45,7 @@ After setting up the tools, set the Karpenter and Kubernetes version: ```bash export KARPENTER_NAMESPACE="kube-system" -export KARPENTER_VERSION="0.35.4" +export KARPENTER_VERSION="0.35.5" export K8S_VERSION="1.29" ``` @@ -87,6 +87,9 @@ The following cluster configuration will: {{% /tab %}} {{< /tabpane >}} +Unless your AWS account has already onboarded to EC2 Spot, you will need to create the service linked role to +avoid the [`ServiceLinkedRoleCreationNotPermitted` error]({{}}). + {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh" language="bash"%}} {{% alert title="Windows Support Notice" color="warning" %}} diff --git a/website/content/en/v0.35/getting-started/getting-started-with-karpenter/cloudformation.yaml b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/cloudformation.yaml index 842594aa1102..1878cd6d352a 100644 --- a/website/content/en/v0.35/getting-started/getting-started-with-karpenter/cloudformation.yaml +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/cloudformation.yaml @@ -213,7 +213,7 @@ Resources: { "Sid": "AllowScopedInstanceProfileCreationActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:CreateInstanceProfile" ], @@ -230,7 +230,7 @@ Resources: { "Sid": "AllowScopedInstanceProfileTagActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:TagInstanceProfile" ], @@ -250,7 +250,7 @@ Resources: { "Sid": "AllowScopedInstanceProfileActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:AddRoleToInstanceProfile", "iam:RemoveRoleFromInstanceProfile", @@ -269,7 +269,7 @@ Resources: { "Sid": "AllowInstanceProfileReadActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": "iam:GetInstanceProfile" }, { @@ -301,6 +301,14 @@ Resources: - sqs.amazonaws.com Action: sqs:SendMessage Resource: !GetAtt KarpenterInterruptionQueue.Arn + - Sid: DenyHTTP + Effect: Deny + Action: sqs:* + Resource: !GetAtt KarpenterInterruptionQueue.Arn + Condition: + Bool: + aws:SecureTransport: false + Principal: "*" ScheduledChangeRule: Type: 'AWS::Events::Rule' Properties: diff --git a/website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json index d474d01f4e16..7f93053b3206 100644 --- a/website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json @@ -1,1440 +1,1698 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 2, - "id": 6, - "links": [], - "liveNow": true, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(action, cluster) (karpenter_deprovisioning_actions_performed)", - "format": "time_series", - "instant": false, - "legendFormat": "{{cluster}}: {{action}}", - "range": true, - "refId": "A" - } - ], - "title": "Deprovisioning Actions Performed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "sum by(cluster) (karpenter_nodes_created)", - "format": "time_series", - "legendFormat": "{{cluster}}", - "range": true, - "refId": "A" - } - ], - "title": "Nodes Created", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 15, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "sum by(cluster) (karpenter_nodes_terminated)", - "format": "time_series", - "legendFormat": "{{cluster}}", - "range": true, - "refId": "A" - } - ], - "title": "Nodes Terminated", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by(phase)(karpenter_pods_state)", - "legendFormat": "{{label_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Pod Phase", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", - "legendFormat": "{{label_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Pod Distribution: $distribution_filter", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-RdYlGr" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*Utilization$" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 1 - }, - { - "id": "unit", - "value": "percentunit" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Provisioned" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 18, - "x": 0, - "y": 29 - }, - "id": 10, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "9.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "karpenter_nodepool_usage{resource_type=\"cpu\"} / karpenter_nodepool_limit{resource_type=\"cpu\"}", - "format": "table", - "instant": true, - "legendFormat": "CPU Limit Utilization", - "range": false, - "refId": "CPU Limit Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Node Count" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "karpenter_nodepool_usage{resource_type=\"memory\"} / karpenter_nodepool_limit{resource_type=\"memory\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Memory Limit Utilization", - "range": false, - "refId": "Memory Limit Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "CPU Capacity" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Memory Capacity" - } - ], - "title": "Nodepool Summary", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "nodepool" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "__name__": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "job": true, - "job 1": true, - "job 2": true, - "resource_type": true, - "resource_type 1": true, - "resource_type 2": true - }, - "indexByName": { - "Time 1": 6, - "Time 2": 7, - "Time 3": 11, - "Time 4": 15, - "Time 5": 16, - "Value #CPU Capacity": 2, - "Value #CPU Limit Utilization": 3, - "Value #Memory Capacity": 4, - "Value #Memory Limit Utilization": 5, - "Value #Node Count": 1, - "instance 1": 8, - "instance 2": 12, - "job 1": 9, - "job 2": 13, - "nodepool": 0, - "resource_type 1": 10, - "resource_type 2": 14 - }, - "renameByName": { - "Time 1": "", - "Value": "CPU Utilization", - "Value #CPU Capacity": "CPU Provisioned", - "Value #CPU Limit Utilization": "CPU Limit Utilization", - "Value #CPU Utilization": "CPU Limit Utilization", - "Value #Memory Capacity": "Memory Provisioned", - "Value #Memory Limit Utilization": "Memory Limit Utilization", - "Value #Memory Utilization": "Memory Utilization", - "Value #Node Count": "Node Count", - "instance": "", - "instance 1": "", - "job": "", - "nodepool": "Nodepool" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 18, - "y": 29 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", - "legendFormat": "Percentage", - "range": true, - "refId": "A" - } - ], - "title": "Spot Node Percentage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-RdYlGr" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "node_name" - }, - "properties": [ - { - "id": "custom.width", - "value": 333 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Utilization" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "thresholds", - "value": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 75 - } - ] - } - }, - { - "id": "max", - "value": 1 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "unit", - "value": "s" - }, - { - "id": "decimals", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 4, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Uptime" - } - ] - }, - "pluginVersion": "9.0.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "CPU Utilization", - "range": false, - "refId": "CPU Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Memory Utilization", - "range": false, - "refId": "Memory Utilization" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Memory Utilization", - "range": false, - "refId": "Pod Count" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\n sum by (node)(node_time_seconds) - sum by (node)(node_boot_time_seconds),\n \"node_name\", \"$1\", \"node\", \"(.+)\"\n)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Uptime", - "range": false, - "refId": "Uptime" - } - ], - "title": "Node Summary", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "node_name" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Value": false, - "Value #Pod Count": false, - "__name__": true, - "arch": true, - "arch 1": true, - "arch 2": true, - "arch 3": true, - "capacity_type 2": true, - "capacity_type 3": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "instance 3": true, - "instance_category 1": true, - "instance_category 2": true, - "instance_category 3": true, - "instance_cpu": true, - "instance_cpu 1": true, - "instance_cpu 2": true, - "instance_cpu 3": true, - "instance_family": true, - "instance_family 1": true, - "instance_family 2": true, - "instance_family 3": true, - "instance_generation 1": true, - "instance_generation 2": true, - "instance_generation 3": true, - "instance_gpu_count": true, - "instance_gpu_count 1": true, - "instance_gpu_count 2": true, - "instance_gpu_count 3": true, - "instance_gpu_manufacturer": true, - "instance_gpu_manufacturer 1": true, - "instance_gpu_manufacturer 2": true, - "instance_gpu_manufacturer 3": true, - "instance_gpu_memory": true, - "instance_gpu_memory 1": true, - "instance_gpu_memory 2": true, - "instance_gpu_memory 3": true, - "instance_gpu_name": true, - "instance_gpu_name 1": true, - "instance_gpu_name 2": true, - "instance_gpu_name 3": true, - "instance_hypervisor": true, - "instance_hypervisor 1": true, - "instance_hypervisor 2": true, - "instance_hypervisor 3": true, - "instance_local_nvme 1": true, - "instance_local_nvme 2": true, - "instance_local_nvme 3": true, - "instance_memory": true, - "instance_memory 1": true, - "instance_memory 2": true, - "instance_memory 3": true, - "instance_pods": true, - "instance_pods 1": true, - "instance_pods 2": true, - "instance_pods 3": true, - "instance_size": true, - "instance_size 1": true, - "instance_size 2": true, - "instance_size 3": true, - "instance_type 1": false, - "instance_type 2": true, - "instance_type 3": true, - "job": true, - "job 1": true, - "job 2": true, - "job 3": true, - "node": true, - "os": true, - "os 1": true, - "os 2": true, - "os 3": true, - "nodepool 1": false, - "nodepool 2": true, - "nodepool 3": true, - "resource_type": true, - "resource_type 1": true, - "resource_type 2": true, - "resource_type 3": true, - "zone 1": false, - "zone 2": true, - "zone 3": true - }, - "indexByName": { - "Time 1": 1, - "Time 2": 25, - "Time 3": 45, - "Time 4": 65, - "Value #CPU Utilization": 10, - "Value #Memory Utilization": 11, - "Value #Pod Count": 9, - "Value #Uptime": 8, - "arch 1": 5, - "arch 2": 26, - "arch 3": 46, - "capacity_type 1": 6, - "capacity_type 2": 27, - "capacity_type 3": 47, - "instance 1": 4, - "instance 2": 28, - "instance 3": 48, - "instance_cpu 1": 12, - "instance_cpu 2": 29, - "instance_cpu 3": 49, - "instance_family 1": 13, - "instance_family 2": 30, - "instance_family 3": 50, - "instance_gpu_count 1": 14, - "instance_gpu_count 2": 31, - "instance_gpu_count 3": 51, - "instance_gpu_manufacturer 1": 15, - "instance_gpu_manufacturer 2": 32, - "instance_gpu_manufacturer 3": 52, - "instance_gpu_memory 1": 16, - "instance_gpu_memory 2": 33, - "instance_gpu_memory 3": 53, - "instance_gpu_name 1": 17, - "instance_gpu_name 2": 34, - "instance_gpu_name 3": 54, - "instance_hypervisor 1": 18, - "instance_hypervisor 2": 35, - "instance_hypervisor 3": 55, - "instance_memory 1": 19, - "instance_memory 2": 36, - "instance_memory 3": 56, - "instance_pods 1": 20, - "instance_pods 2": 37, - "instance_pods 3": 57, - "instance_size 1": 21, - "instance_size 2": 38, - "instance_size 3": 58, - "instance_type 1": 3, - "instance_type 2": 39, - "instance_type 3": 59, - "job 1": 22, - "job 2": 40, - "job 3": 60, - "node": 66, - "node_name": 0, - "os 1": 23, - "os 2": 41, - "os 3": 61, - "nodepool 1": 2, - "nodepool 2": 42, - "nodepool 3": 62, - "resource_type 1": 24, - "resource_type 2": 43, - "resource_type 3": 63, - "zone 1": 7, - "zone 2": 44, - "zone 3": 64 - }, - "renameByName": { - "Time": "", - "Time 1": "", - "Value": "CPU Utilization", - "Value #Allocatable": "", - "Value #CPU Utilization": "CPU Utilization", - "Value #Memory Utilization": "Memory Utilization", - "Value #Pod CPU": "", - "Value #Pod Count": "Pods", - "Value #Uptime": "Uptime", - "arch": "Architecture", - "arch 1": "Arch", - "capacity_type": "Capacity Type", - "capacity_type 1": "Capacity Type", - "instance 1": "Instance", - "instance_cpu 1": "vCPU", - "instance_type": "Instance Type", - "instance_type 1": "Instance Type", - "node_name": "Node Name", - "nodepool 1": "Nodepool", - "zone 1": "Zone" - } - } - } - ], - "type": "table" - } - ], - "refresh": false, - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "includeAll": false, - "label": "Data Source", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, nodepool)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "nodepool", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, nodepool)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, zone)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "zone", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, zone)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, arch)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "arch", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, arch)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, capacity_type)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "capacity_type", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, capacity_type)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(karpenter_nodes_allocatable, instance_type)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "instance_type", - "options": [], - "query": { - "query": "label_values(karpenter_nodes_allocatable, instance_type)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": "nodepool", - "value": "nodepool" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "distribution_filter", - "options": [ - { - "selected": false, - "text": "arch", - "value": "arch" - }, - { - "selected": false, - "text": "capacity_type", - "value": "capacity_type" - }, - { - "selected": false, - "text": "instance_type", - "value": "instance_type" - }, - { - "selected": false, - "text": "namespace", - "value": "namespace" - }, - { - "selected": false, - "text": "node", - "value": "node" - }, - { - "selected": true, - "text": "nodepool", - "value": "nodepool" - }, - { - "selected": false, - "text": "zone", - "value": "zone" - } - ], - "query": "arch,capacity_type,instance_type,namespace,node,nodepool,zone", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Karpenter Capacity", - "uid": "ta8I9Q67z", - "version": 4, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 2, + "id": 44, + "links": [], + "liveNow": true, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(cluster,nodepool) (karpenter_nodes_created{nodepool=~\"$nodepool\"})", + "format": "time_series", + "legendFormat": "{{cluster}}", + "range": true, + "refId": "A" + } + ], + "title": "Nodes Created: nodepool \"$nodepool\"", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(cluster,nodepool) (karpenter_nodes_terminated{nodepool=~\"$nodepool\"})", + "format": "time_series", + "legendFormat": "{{cluster}}", + "range": true, + "refId": "A" + } + ], + "title": "Nodes Terminated: nodepool \"$nodepool\"", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(consolidation_type,method)(karpenter_disruption_eligible_nodes)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Nodes Eligible for Disruptions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(action,consolidation_type,method)(karpenter_disruption_actions_performed_total)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Disruption Actions Performed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "See: https://karpenter.sh/v0.35/concepts/disruption/#automated-methods", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(action,consolidation_type,method)(karpenter_disruption_nodes_disrupted_total{nodepool=~\"$nodepool\"})", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Voluntary Node Disruptions: nodepool \"$nodepool\"", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pod Distribution: $distribution_filter", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(phase)(karpenter_pods_state)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pod Phase", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*Utilization$" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 1 + }, + { + "id": "unit", + "value": "percentunit" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Provisioned" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 18, + "x": 0, + "y": 42 + }, + "id": 10, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "karpenter_nodepool_usage{resource_type=\"cpu\"} / karpenter_nodepool_limit{resource_type=\"cpu\"}", + "format": "table", + "instant": true, + "legendFormat": "CPU Limit Utilization", + "range": false, + "refId": "CPU Limit Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "Node Count" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "karpenter_nodepool_usage{resource_type=\"memory\"} / karpenter_nodepool_limit{resource_type=\"memory\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Memory Limit Utilization", + "range": false, + "refId": "Memory Limit Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "CPU Capacity" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "Memory Capacity" + } + ], + "title": "Nodepool Summary", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "nodepool" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "__name__": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "job": true, + "job 1": true, + "job 2": true, + "resource_type": true, + "resource_type 1": true, + "resource_type 2": true + }, + "indexByName": { + "Time 1": 6, + "Time 2": 7, + "Time 3": 11, + "Time 4": 15, + "Time 5": 16, + "Value #CPU Capacity": 2, + "Value #CPU Limit Utilization": 3, + "Value #Memory Capacity": 4, + "Value #Memory Limit Utilization": 5, + "Value #Node Count": 1, + "instance 1": 8, + "instance 2": 12, + "job 1": 9, + "job 2": 13, + "nodepool": 0, + "resource_type 1": 10, + "resource_type 2": 14 + }, + "renameByName": { + "Time 1": "", + "Value": "CPU Utilization", + "Value #CPU Capacity": "CPU Provisioned", + "Value #CPU Limit Utilization": "CPU Limit Utilization", + "Value #CPU Utilization": "CPU Limit Utilization", + "Value #Memory Capacity": "Memory Provisioned", + "Value #Memory Limit Utilization": "Memory Limit Utilization", + "Value #Memory Utilization": "Memory Utilization", + "Value #Node Count": "Node Count", + "instance": "", + "instance 1": "", + "job": "", + "nodepool": "Nodepool" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 18, + "y": 42 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", + "legendFormat": "Percentage", + "range": true, + "refId": "A" + } + ], + "title": "Spot Node Percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "node_name" + }, + "properties": [ + { + "id": "custom.width", + "value": 333 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Utilization" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "percentage", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 75 + } + ] + } + }, + { + "id": "max", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Uptime" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "decimals", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Uptime" + } + ] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "CPU Utilization", + "range": false, + "refId": "CPU Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Memory Utilization", + "range": false, + "refId": "Memory Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Memory Utilization", + "range": false, + "refId": "Pod Count" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\n sum by (node)(node_time_seconds) - sum by (node)(node_boot_time_seconds),\n \"node_name\", \"$1\", \"node\", \"(.+)\"\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Uptime", + "range": false, + "refId": "Uptime" + } + ], + "title": "Node Summary", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "node_name" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Value": false, + "Value #Pod Count": false, + "__name__": true, + "arch": true, + "arch 1": true, + "arch 2": true, + "arch 3": true, + "capacity_type 2": true, + "capacity_type 3": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "instance 3": true, + "instance_category 1": true, + "instance_category 2": true, + "instance_category 3": true, + "instance_cpu": true, + "instance_cpu 1": true, + "instance_cpu 2": true, + "instance_cpu 3": true, + "instance_family": true, + "instance_family 1": true, + "instance_family 2": true, + "instance_family 3": true, + "instance_generation 1": true, + "instance_generation 2": true, + "instance_generation 3": true, + "instance_gpu_count": true, + "instance_gpu_count 1": true, + "instance_gpu_count 2": true, + "instance_gpu_count 3": true, + "instance_gpu_manufacturer": true, + "instance_gpu_manufacturer 1": true, + "instance_gpu_manufacturer 2": true, + "instance_gpu_manufacturer 3": true, + "instance_gpu_memory": true, + "instance_gpu_memory 1": true, + "instance_gpu_memory 2": true, + "instance_gpu_memory 3": true, + "instance_gpu_name": true, + "instance_gpu_name 1": true, + "instance_gpu_name 2": true, + "instance_gpu_name 3": true, + "instance_hypervisor": true, + "instance_hypervisor 1": true, + "instance_hypervisor 2": true, + "instance_hypervisor 3": true, + "instance_local_nvme 1": true, + "instance_local_nvme 2": true, + "instance_local_nvme 3": true, + "instance_memory": true, + "instance_memory 1": true, + "instance_memory 2": true, + "instance_memory 3": true, + "instance_pods": true, + "instance_pods 1": true, + "instance_pods 2": true, + "instance_pods 3": true, + "instance_size": true, + "instance_size 1": true, + "instance_size 2": true, + "instance_size 3": true, + "instance_type 1": false, + "instance_type 2": true, + "instance_type 3": true, + "job": true, + "job 1": true, + "job 2": true, + "job 3": true, + "node": true, + "nodepool 1": false, + "nodepool 2": true, + "nodepool 3": true, + "os": true, + "os 1": true, + "os 2": true, + "os 3": true, + "resource_type": true, + "resource_type 1": true, + "resource_type 2": true, + "resource_type 3": true, + "zone 1": false, + "zone 2": true, + "zone 3": true + }, + "indexByName": { + "Time 1": 1, + "Time 2": 25, + "Time 3": 45, + "Time 4": 65, + "Value #CPU Utilization": 10, + "Value #Memory Utilization": 11, + "Value #Pod Count": 9, + "Value #Uptime": 8, + "arch 1": 5, + "arch 2": 26, + "arch 3": 46, + "capacity_type 1": 6, + "capacity_type 2": 27, + "capacity_type 3": 47, + "instance 1": 4, + "instance 2": 28, + "instance 3": 48, + "instance_cpu 1": 12, + "instance_cpu 2": 29, + "instance_cpu 3": 49, + "instance_family 1": 13, + "instance_family 2": 30, + "instance_family 3": 50, + "instance_gpu_count 1": 14, + "instance_gpu_count 2": 31, + "instance_gpu_count 3": 51, + "instance_gpu_manufacturer 1": 15, + "instance_gpu_manufacturer 2": 32, + "instance_gpu_manufacturer 3": 52, + "instance_gpu_memory 1": 16, + "instance_gpu_memory 2": 33, + "instance_gpu_memory 3": 53, + "instance_gpu_name 1": 17, + "instance_gpu_name 2": 34, + "instance_gpu_name 3": 54, + "instance_hypervisor 1": 18, + "instance_hypervisor 2": 35, + "instance_hypervisor 3": 55, + "instance_memory 1": 19, + "instance_memory 2": 36, + "instance_memory 3": 56, + "instance_pods 1": 20, + "instance_pods 2": 37, + "instance_pods 3": 57, + "instance_size 1": 21, + "instance_size 2": 38, + "instance_size 3": 58, + "instance_type 1": 3, + "instance_type 2": 39, + "instance_type 3": 59, + "job 1": 22, + "job 2": 40, + "job 3": 60, + "node": 66, + "node_name": 0, + "nodepool 1": 2, + "nodepool 2": 42, + "nodepool 3": 62, + "os 1": 23, + "os 2": 41, + "os 3": 61, + "resource_type 1": 24, + "resource_type 2": 43, + "resource_type 3": 63, + "zone 1": 7, + "zone 2": 44, + "zone 3": 64 + }, + "renameByName": { + "Time": "", + "Time 1": "", + "Value": "CPU Utilization", + "Value #Allocatable": "", + "Value #CPU Utilization": "CPU Utilization", + "Value #Memory Utilization": "Memory Utilization", + "Value #Pod CPU": "", + "Value #Pod Count": "Pods", + "Value #Uptime": "Uptime", + "arch": "Architecture", + "arch 1": "Arch", + "capacity_type": "Capacity Type", + "capacity_type 1": "Capacity Type", + "instance 1": "Instance", + "instance_cpu 1": "vCPU", + "instance_type": "Instance Type", + "instance_type 1": "Instance Type", + "node_name": "Node Name", + "nodepool 1": "Nodepool", + "zone 1": "Zone" + } + } + } + ], + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, nodepool)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "nodepool", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, nodepool)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, zone)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "zone", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, zone)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, arch)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "arch", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, arch)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, capacity_type)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "capacity_type", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, capacity_type)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, instance_type)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "instance_type", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, instance_type)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(karpenter_disruption_actions_performed_total,method)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(karpenter_disruption_actions_performed_total,method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "nodepool", + "value": "nodepool" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "distribution_filter", + "options": [ + { + "selected": false, + "text": "arch", + "value": "arch" + }, + { + "selected": false, + "text": "capacity_type", + "value": "capacity_type" + }, + { + "selected": false, + "text": "instance_type", + "value": "instance_type" + }, + { + "selected": false, + "text": "method", + "value": "method" + }, + { + "selected": false, + "text": "namespace", + "value": "namespace" + }, + { + "selected": false, + "text": "node", + "value": "node" + }, + { + "selected": true, + "text": "nodepool", + "value": "nodepool" + }, + { + "selected": false, + "text": "zone", + "value": "zone" + } + ], + "query": "arch,capacity_type,instance_type,method,namespace,node,nodepool,zone", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Karpenter Capacity", + "uid": "ta8I9Q67z", + "version": 5, + "weekStart": "" } diff --git a/website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json index c7762d302f99..174282ee9468 100644 --- a/website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json @@ -498,7 +498,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(controller_runtime_reconcile_total[10m])) by (controller)", + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"karpenter\"}[10m])) by (controller)", "legendFormat": "{{controller}}", "range": true, "refId": "A" @@ -542,14 +542,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(controller_runtime_reconcile_time_seconds_count, controller)", + "definition": "label_values(controller_runtime_reconcile_time_seconds_count{job=\"karpenter\"}, controller)", "hide": 0, "includeAll": false, "multi": false, "name": "controller", "options": [], "query": { - "query": "label_values(controller_runtime_reconcile_time_seconds_count, controller)", + "query": "label_values(controller_runtime_reconcile_time_seconds_count{job=\"karpenter\"}, controller)", "refId": "StandardVariableQuery" }, "refresh": 2, diff --git a/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh index c17584cec718..0854100d516c 100755 --- a/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh @@ -14,12 +14,18 @@ spec: app: inflate spec: terminationGracePeriodSeconds: 0 + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 containers: - - name: inflate - image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 - resources: - requests: - cpu: 1 + - name: inflate + image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 + resources: + requests: + cpu: 1 + securityContext: + allowPrivilegeEscalation: false EOF kubectl scale deployment inflate --replicas 5 diff --git a/website/content/en/v0.35/getting-started/migrating-from-cas/_index.md b/website/content/en/v0.35/getting-started/migrating-from-cas/_index.md index 398bf41e3eb8..6ac44da937dd 100644 --- a/website/content/en/v0.35/getting-started/migrating-from-cas/_index.md +++ b/website/content/en/v0.35/getting-started/migrating-from-cas/_index.md @@ -92,7 +92,7 @@ One for your Karpenter node role and one for your existing node group. First set the Karpenter release you want to deploy. ```bash -export KARPENTER_VERSION="0.35.4" +export KARPENTER_VERSION="0.35.5" ``` We can now generate a full Karpenter deployment yaml from the Helm chart. @@ -117,7 +117,6 @@ affinity: - matchExpressions: - key: karpenter.sh/nodepool operator: DoesNotExist - - matchExpressions: - key: eks.amazonaws.com/nodegroup operator: In values: @@ -133,7 +132,7 @@ Now that our deployment is ready we can create the karpenter namespace, create t ## Create default NodePool -We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.35.4/examples/v1beta1) for specific needs. +We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.35.5/examples/v1beta1) for specific needs. {{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh" language="bash" %}} diff --git a/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh index 47df188dc87d..31e8093cd6f8 100644 --- a/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh +++ b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh @@ -1,6 +1,6 @@ for NODEGROUP in $(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" --query 'nodegroups' --output text); do aws ec2 create-tags \ --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ - --resources "$(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" \ - --nodegroup-name "${NODEGROUP}" --query 'nodegroup.subnets' --output text )" + --resources $(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" \ + --nodegroup-name "${NODEGROUP}" --query 'nodegroup.subnets' --output text ) done diff --git a/website/content/en/v0.35/reference/cloudformation.md b/website/content/en/v0.35/reference/cloudformation.md index 6811706dbea7..425deca20bd5 100644 --- a/website/content/en/v0.35/reference/cloudformation.md +++ b/website/content/en/v0.35/reference/cloudformation.md @@ -17,7 +17,7 @@ These descriptions should allow you to understand: To download a particular version of `cloudformation.yaml`, set the version and use `curl` to pull the file to your local system: ```bash -export KARPENTER_VERSION="0.35.4" +export KARPENTER_VERSION="0.35.5" curl https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > cloudformation.yaml ``` @@ -376,14 +376,14 @@ This gives EC2 permission explicit permission to use the `KarpenterNodeRole-${Cl #### AllowScopedInstanceProfileCreationActions The AllowScopedInstanceProfileCreationActions Sid gives the Karpenter controller permission to create a new instance profile with [`iam:CreateInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_CreateInstanceProfile.html), -provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName` set to owned and is made in the current region. +provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName}` set to owned and is made in the current region. Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures that Karpenter can generate instance profiles on your behalf based on roles specified in your `EC2NodeClasses` that you use to configure Karpenter. ```json { "Sid": "AllowScopedInstanceProfileCreationActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:CreateInstanceProfile" ], @@ -408,7 +408,7 @@ Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures t { "Sid": "AllowScopedInstanceProfileTagActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:TagInstanceProfile" ], @@ -431,14 +431,14 @@ Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures t #### AllowScopedInstanceProfileActions The AllowScopedInstanceProfileActions Sid gives the Karpenter controller permission to perform [`iam:AddRoleToInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_AddRoleToInstanceProfile.html), [`iam:RemoveRoleFromInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_RemoveRoleFromInstanceProfile.html), and [`iam:DeleteInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_DeleteInstanceProfile.html) actions, -provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName` set to owned and is made in the current region. +provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName}` set to owned and is made in the current region. Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This permission is further enforced by the `iam:PassRole` permission. If Karpenter attempts to add a role to an instance profile that it doesn't have `iam:PassRole` permission on, that call will fail. Therefore, if you configure Karpenter to use a new role through the `EC2NodeClass`, ensure that you also specify that role within your `iam:PassRole` permission. ```json { "Sid": "AllowScopedInstanceProfileActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*", "Action": [ "iam:AddRoleToInstanceProfile", "iam:RemoveRoleFromInstanceProfile", @@ -464,7 +464,7 @@ The AllowInstanceProfileActions Sid gives the Karpenter controller permission to { "Sid": "AllowInstanceProfileReadActions", "Effect": "Allow", - "Resource": "*", + "Resource": "arn:aws:iam::${AWS::AccountId}:instance-profile/*", "Action": "iam:GetInstanceProfile" } ``` @@ -525,6 +525,7 @@ KarpenterInterruptionQueue: The Karpenter interruption queue policy is created to allow AWS services that we want to receive instance notifications from to push notification messages to the queue. The [AWS::SQS::QueuePolicy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sqs-queuepolicy.html) resource here applies `EC2InterruptionPolicy` to the `KarpenterInterruptionQueue`. The policy allows [sqs:SendMessage](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html) actions to `events.amazonaws.com` and `sqs.amazonaws.com` services. It also allows the `GetAtt` function to get attributes from `KarpenterInterruptionQueue.Arn`. +Additionally, it only allows access to the queue using encrypted connections over HTTPS (TLS) to adhere to [Amazon SQS Security Best Practices](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-security-best-practices.html#enforce-encryption-data-in-transit). ```yaml KarpenterInterruptionQueuePolicy: @@ -542,6 +543,14 @@ KarpenterInterruptionQueuePolicy: - sqs.amazonaws.com Action: sqs:SendMessage Resource: !GetAtt KarpenterInterruptionQueue.Arn + - Sid: DenyHTTP + Effect: Deny + Action: sqs:* + Resource: !GetAtt KarpenterInterruptionQueue.Arn + Condition: + Bool: + aws:SecureTransport: false + Principal: "*" ``` ### Rules diff --git a/website/content/en/v0.35/reference/instance-types.md b/website/content/en/v0.35/reference/instance-types.md index 6978c20ef258..f93fc2c75a81 100644 --- a/website/content/en/v0.35/reference/instance-types.md +++ b/website/content/en/v0.35/reference/instance-types.md @@ -10,8 +10,7 @@ description: > AWS instance types offer varying resources and can be selected by labels. The values provided below are the resources available with some assumptions and after the instance overhead has been subtracted: - `blockDeviceMappings` are not configured -- `aws-eni-limited-pod-density` is assumed to be `true` -- `amiFamily` is set to the default of `AL2` +- `amiFamily` is set to `AL2023` ## a1 Family ### `a1.medium` #### Labels @@ -3049,8 +3048,8 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|238333Mi| - |pods|345| + |memory|237794Mi| + |pods|394| |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `c6in.metal` @@ -3074,8 +3073,8 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|238333Mi| - |pods|345| + |memory|237794Mi| + |pods|394| |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ## c7a Family @@ -3790,6 +3789,30 @@ below are the resources available with some assumptions and after the instance o |pods|737| |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| +### `c7gd.metal` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-local-nvme|3800| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-size|metal| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7gd.metal| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|112720Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| ## c7gn Family ### `c7gn.medium` #### Labels @@ -3984,6 +4007,29 @@ below are the resources available with some assumptions and after the instance o |pods|737| |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| +### `c7gn.metal` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7gn| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-size|metal| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7gn.metal| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|112720Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| ## c7i Family ### `c7i.large` #### Labels @@ -5598,6 +5644,294 @@ below are the resources available with some assumptions and after the instance o |nvidia.com/gpu|2| |pods|737| |vpc.amazonaws.com/pod-eni|107| +## g6 Family +### `g6.xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|g| + |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|g6| + |karpenter.k8s.aws/instance-generation|6| + |karpenter.k8s.aws/instance-gpu-count|1| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|22888| + |karpenter.k8s.aws/instance-gpu-name|l4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|250| + |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-size|xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|g6.xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|3920m| + |ephemeral-storage|17Gi| + |memory|14162Mi| + |nvidia.com/gpu|1| + |pods|58| +### `g6.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|g| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|g6| + |karpenter.k8s.aws/instance-generation|6| + |karpenter.k8s.aws/instance-gpu-count|1| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|22888| + |karpenter.k8s.aws/instance-gpu-name|l4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|450| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|g6.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|29317Mi| + |nvidia.com/gpu|1| + |pods|58| +### `g6.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|g| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|g6| + |karpenter.k8s.aws/instance-generation|6| + |karpenter.k8s.aws/instance-gpu-count|1| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|22888| + |karpenter.k8s.aws/instance-gpu-name|l4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|600| + |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|g6.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|57691Mi| + |nvidia.com/gpu|1| + |pods|234| +### `g6.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|g| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|g6| + |karpenter.k8s.aws/instance-generation|6| + |karpenter.k8s.aws/instance-gpu-count|1| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|22888| + |karpenter.k8s.aws/instance-gpu-name|l4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|900| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|g6.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|118312Mi| + |nvidia.com/gpu|1| + |pods|234| + |vpc.amazonaws.com/efa|1| +### `g6.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|g| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|g6| + |karpenter.k8s.aws/instance-generation|6| + |karpenter.k8s.aws/instance-gpu-count|4| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|91553| + |karpenter.k8s.aws/instance-gpu-name|l4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|15200| + |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|g6.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|178933Mi| + |nvidia.com/gpu|4| + |pods|234| + |vpc.amazonaws.com/efa|1| +### `g6.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|g| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|g6| + |karpenter.k8s.aws/instance-generation|6| + |karpenter.k8s.aws/instance-gpu-count|1| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|22888| + |karpenter.k8s.aws/instance-gpu-name|l4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|3800| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|g6.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|234021Mi| + |nvidia.com/gpu|1| + |pods|737| + |vpc.amazonaws.com/efa|1| +### `g6.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|g| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|g6| + |karpenter.k8s.aws/instance-generation|6| + |karpenter.k8s.aws/instance-gpu-count|4| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|91553| + |karpenter.k8s.aws/instance-gpu-name|l4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|15200| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|g6.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|355262Mi| + |nvidia.com/gpu|4| + |pods|737| + |vpc.amazonaws.com/efa|1| +### `g6.48xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|g| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|g6| + |karpenter.k8s.aws/instance-generation|6| + |karpenter.k8s.aws/instance-gpu-count|8| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|183105| + |karpenter.k8s.aws/instance-gpu-name|l4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|60800| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-size|48xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|g6.48xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|718987Mi| + |nvidia.com/gpu|8| + |pods|737| + |vpc.amazonaws.com/efa|1| +## gr6 Family +### `gr6.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|gr| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|gr6| + |karpenter.k8s.aws/instance-generation|6| + |karpenter.k8s.aws/instance-gpu-count|1| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|22888| + |karpenter.k8s.aws/instance-gpu-name|l4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|600| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|gr6.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|118312Mi| + |nvidia.com/gpu|1| + |pods|234| +### `gr6.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|gr| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|gr6| + |karpenter.k8s.aws/instance-generation|6| + |karpenter.k8s.aws/instance-gpu-count|1| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|22888| + |karpenter.k8s.aws/instance-gpu-name|l4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|900| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|gr6.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|239554Mi| + |nvidia.com/gpu|1| + |pods|234| + |vpc.amazonaws.com/efa|1| ## h1 Family ### `h1.2xlarge` #### Labels @@ -10426,8 +10760,8 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|480816Mi| - |pods|345| + |memory|480277Mi| + |pods|394| |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `m6idn.metal` @@ -10452,8 +10786,8 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|480816Mi| - |pods|345| + |memory|480277Mi| + |pods|394| |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ## m6in Family @@ -10670,8 +11004,8 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|480816Mi| - |pods|345| + |memory|480277Mi| + |pods|394| |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `m6in.metal` @@ -10695,8 +11029,8 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|480816Mi| - |pods|345| + |memory|480277Mi| + |pods|394| |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ## m7a Family @@ -15268,8 +15602,8 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|965782Mi| - |pods|345| + |memory|965243Mi| + |pods|394| |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `r6idn.metal` @@ -15294,8 +15628,8 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|965782Mi| - |pods|345| + |memory|965243Mi| + |pods|394| |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ## r6in Family @@ -15512,8 +15846,8 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|965782Mi| - |pods|345| + |memory|965243Mi| + |pods|394| |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `r6in.metal` @@ -15537,8 +15871,8 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|127610m| |ephemeral-storage|17Gi| - |memory|965782Mi| - |pods|345| + |memory|965243Mi| + |pods|394| |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ## r7a Family @@ -16253,6 +16587,30 @@ below are the resources available with some assumptions and after the instance o |pods|737| |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| +### `r7gd.metal` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-local-nvme|3800| + |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-size|metal| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.metal| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|476445Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| ## r7i Family ### `r7i.large` #### Labels diff --git a/website/content/en/v0.35/reference/threat-model.md b/website/content/en/v0.35/reference/threat-model.md index 88f735fb3215..5f932e72127b 100644 --- a/website/content/en/v0.35/reference/threat-model.md +++ b/website/content/en/v0.35/reference/threat-model.md @@ -31,11 +31,11 @@ A Cluster Developer has the ability to create pods via `Deployments`, `ReplicaSe Karpenter has permissions to create and manage cloud instances. Karpenter has Kubernetes API permissions to create, update, and remove nodes, as well as evict pods. For a full list of the permissions, see the RBAC rules in the helm chart template. Karpenter also has AWS IAM permissions to create instances with IAM roles. -* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/aggregate-clusterrole.yaml) -* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/clusterrole-core.yaml) -* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/clusterrole.yaml) -* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/rolebinding.yaml) -* [role.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/role.yaml) +* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.35.5/charts/karpenter/templates/aggregate-clusterrole.yaml) +* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.35.5/charts/karpenter/templates/clusterrole-core.yaml) +* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.35.5/charts/karpenter/templates/clusterrole.yaml) +* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.35.5/charts/karpenter/templates/rolebinding.yaml) +* [role.yaml](https://github.com/aws/karpenter/blob/v0.35.5/charts/karpenter/templates/role.yaml) ## Assumptions diff --git a/website/content/en/v0.35/troubleshooting.md b/website/content/en/v0.35/troubleshooting.md index 6dc784007b75..6b362e504e86 100644 --- a/website/content/en/v0.35/troubleshooting.md +++ b/website/content/en/v0.35/troubleshooting.md @@ -73,7 +73,7 @@ Node role names for Karpenter are created in the form `KarpenterNodeRole-${Clust If a long cluster name causes the Karpenter node role name to exceed 64 characters, creating that object will fail. Keep in mind that `KarpenterNodeRole-` is just a recommendation from the getting started guide. -Instead using of the eksctl role, you can shorten the name to anything you like, as long as it has the right permissions. +Instead of using the eksctl role, you can shorten the name to anything you like, as long as it has the right permissions. ### Unknown field in Provisioner spec @@ -123,8 +123,9 @@ kubectl label crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodecl - In the case of `annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "karpenter"` run: ```shell +KARPENTER_NAMESPACE=kube-system kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite -kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace="${KARPENTER_NAMESPACE}" --overwrite ``` ## Uninstallation @@ -330,6 +331,11 @@ By default, the number of pods on a node is limited by both the number of networ If the max-pods (configured through your Provisioner [`kubeletConfiguration`]({{}})) is greater than the number of supported IPs for a given instance type, the CNI will fail to assign an IP to the pod and your pod will be left in a `ContainerCreating` state. +If you've enabled [Security Groups per Pod](https://aws.github.io/aws-eks-best-practices/networking/sgpp/), one of the instance's ENIs is reserved as the trunk interface and uses branch interfaces off of that trunk interface to assign different security groups. +If you do not have any `SecurityGroupPolicies` configured for your pods, they will be unable to utilize branch interfaces attached to the trunk interface, and IPs will only be available from the non-trunk ENIs. +This effectively reduces the max-pods value by the number of IPs that would have been available from the trunk ENI. +Note that Karpenter is not aware if [Security Groups per Pod](https://aws.github.io/aws-eks-best-practices/networking/sgpp/) is enabled, and will continue to compute max-pods assuming all ENIs on the instance can be utilized. + ##### Solutions To avoid this discrepancy between `maxPods` and the supported pod density of the EC2 instance based on ENIs and allocatable IPs, you can perform one of the following actions on your cluster: @@ -647,7 +653,7 @@ To correct the problem if it occurs, you can use the approach that AWS EBS uses, "AWS": "arn:aws:iam::${AWS_ACCOUNT_ID}:root" }, "Action": [ - "kms:Describe", + "kms:Describe*", "kms:Get*", "kms:List*", "kms:RevokeGrant" @@ -663,7 +669,7 @@ This typically occurs when the node has not been considered fully initialized fo ### Log message of `inflight check failed for node, Expected resource "vpc.amazonaws.com/pod-eni" didn't register on the node` is reported -This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. If you've enabled Pod ENI for Karpenter nodes via the `aws.enablePodENI` setting, you will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. +This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. You will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. ### AWS Node Termination Handler (NTH) interactions Karpenter [doesn't currently support draining and terminating on spot rebalance recommendations]({{< ref "concepts/disruption#interruption" >}}). Users who want support for both drain and terminate on spot interruption as well as drain and termination on spot rebalance recommendations may install Node Termination Handler (NTH) on their clusters to support this behavior. diff --git a/website/content/en/v0.35/upgrading/compatibility.md b/website/content/en/v0.35/upgrading/compatibility.md index 0f2c8d6cab94..f948450e8319 100644 --- a/website/content/en/v0.35/upgrading/compatibility.md +++ b/website/content/en/v0.35/upgrading/compatibility.md @@ -75,14 +75,13 @@ Karpenter offers three types of releases. This section explains the purpose of e ### Stable Releases -Stable releases are the most reliable releases that are released with weekly cadence. Stable releases are our only recommended versions for production environments. -Sometimes we skip a stable release because we find instability or problems that need to be fixed before having a stable release. -Stable releases are tagged with a semantic version prefixed by a `v`. For example `v0.13.0`. +Stable releases are the only recommended versions for production environments. Stable releases are tagged with a semantic version (e.g. `0.35.0`). Note that stable releases prior to `0.35.0` are prefixed with a `v` (e.g. `v0.34.0`). ### Release Candidates -We consider having release candidates for major and important minor versions. Our release candidates are tagged like `vx.y.z-rc.0`, `vx.y.z-rc.1`. The release candidate will then graduate to `vx.y.z` as a normal stable release. +We consider having release candidates for major and important minor versions. Our release candidates are tagged like `x.y.z-rc.0`, `x.y.z-rc.1`. The release candidate will then graduate to `x.y.z` as a stable release. By adopting this practice we allow our users who are early adopters to test out new releases before they are available to the wider community, thereby providing us with early feedback resulting in more stable releases. +Note that, like the stable releases, release candidates prior to `0.35.0` are prefixed with a `v`. ### Snapshot Releases diff --git a/website/content/en/v0.35/upgrading/upgrade-guide.md b/website/content/en/v0.35/upgrading/upgrade-guide.md index 42acba982977..89c67d12d222 100644 --- a/website/content/en/v0.35/upgrading/upgrade-guide.md +++ b/website/content/en/v0.35/upgrading/upgrade-guide.md @@ -28,9 +28,9 @@ If you get the error `invalid ownership metadata; label validation error:` while In general, you can reapply the CRDs in the `crds` directory of the Karpenter Helm chart: ```shell -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.35.4/pkg/apis/crds/karpenter.sh_nodepools.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.35.4/pkg/apis/crds/karpenter.sh_nodeclaims.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.35.4/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.35.5/pkg/apis/crds/karpenter.sh_nodepools.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.35.5/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.35.5/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml ``` -[comment]: <> (WHEN CREATING A NEW SECTION OF THE UPGRADE GUIDANCE FOR NEWER VERSIONS, ENSURE THAT YOU COPY THE ALERT SECTION BELOW TO PROPERLY WARN USERS OF THE RISK OF UPGRADING WITHOUT GOING TO v0.32 FIRST) +### Upgrading to `0.37.0`+ {{% alert title="Warning" color="warning" %}} -v0.33.0+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to v0.33.0+ without first [upgrading to v0.32.x]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to `0.37.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} + +* Karpenter now adds a readiness status condition to the EC2NodeClass. Make sure to upgrade your Custom Resource Definitions before proceeding with the upgrade. Failure to do so will result in Karpenter being unable to provision new nodes. +* Karpenter no longer updates the logger name when creating controller loggers. We now adhere to the controller-runtime standard, where the logger name will be set as `"logger": "controller"` always and the controller name will be stored in the structured value `"controller"` +* Karpenter updated the NodeClass controller naming in the following way: `nodeclass` -> `nodeclass.status`, `nodeclass.hash`, `nodeclass.termination` +* Karpenter's NodeClaim status conditions no longer include the `severity` field + +### Upgrading to `0.36.0`+ + +{{% alert title="Warning" color="warning" %}} +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to `0.36.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} + +{{% alert title="Warning" color="warning" %}} + v0.36.x introduces update to drift that restricts rollback. When rolling back from >=v0.36.0, note that v0.32.9+, v0.33.4+, v0.34.5+, v0.35.4+ are the patch versions that support rollback. If Karpenter is rolled back to an older patch version, Karpenter can potentially drift all the nodes in the cluster. +{{% /alert %}} + +* Karpenter changed the name of the `karpenter_cloudprovider_instance_type_price_estimate` metric to `karpenter_cloudprovider_instance_type_offering_price_estimate` to align with the new `karpenter_cloudprovider_instance_type_offering_available` metric. The `region` label was also dropped from the metric, since this can be inferred from the environment that Karpenter is running in. + +### Upgrading to `0.35.0`+ + +{{% alert title="Warning" color="warning" %}} +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to `0.35.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} + +* Karpenter OCI tags and Helm chart version are now valid semantic versions, meaning that the `v` prefix from the git tag has been removed and they now follow the `x.y.z` pattern. + +### Upgrading to `0.34.0`+ + +{{% alert title="Warning" color="warning" %}} +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to `0.34.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. {{% /alert %}} {{% alert title="Warning" color="warning" %}} @@ -46,82 +79,81 @@ The Ubuntu EKS optimized AMI has moved from 20.04 to 22.04 for Kubernetes 1.29+. {{% /alert %}} * Karpenter now supports `nodepool.spec.disruption.budgets`, which allows users to control the speed of disruption in the cluster. Since this requires an update to the Custom Resource, before upgrading, you should re-apply the new updates to the CRDs. Check out [Disruption Budgets]({{}}) for more. -* With Disruption Budgets, Karpenter will disrupt multiple batches of nodes simultaneously, which can result in overall quicker scale-down of your cluster. Before v0.34, Karpenter had a hard-coded parallelism limit for each type of disruption. In v0.34, Karpenter will now disrupt at most 10% of nodes for a given NodePool. There is no setting that will be perfectly equivalent with the behavior prior to v0.34. When considering how to configure your budgets, please refer to the following limits for versions prior to v0.34: +* With Disruption Budgets, Karpenter will disrupt multiple batches of nodes simultaneously, which can result in overall quicker scale-down of your cluster. Before `0.34.0`, Karpenter had a hard-coded parallelism limit for each type of disruption. In `0.34.0`+, Karpenter will now disrupt at most 10% of nodes for a given NodePool. There is no setting that will be perfectly equivalent with the behavior prior to `0.34.0`. When considering how to configure your budgets, please refer to the following limits for versions prior to `0.34.0`: * `Empty Expiration / Empty Drift / Empty Consolidation`: infinite parallelism * `Non-Empty Expiration / Non-Empty Drift / Single-Node Consolidation`: one node at a time * `Multi-Node Consolidation`: max 100 nodes -* To support Disruption Budgets, v0.34+ includes critical changes to Karpenter's core controllers, which allows Karpenter to consider multiple batches of disrupting nodes simultaneously. This increases Karpenter's performance with the potential downside of higher CPU and memory utilization from the Karpenter pod. While the magnitude of this difference varies on a case-by-case basis, when upgrading to Karpenter v0.34+, please note that you may need to increase the resources allocated to the Karpenter controller pods. +* To support Disruption Budgets, `0.34.0`+ includes critical changes to Karpenter's core controllers, which allows Karpenter to consider multiple batches of disrupting nodes simultaneously. This increases Karpenter's performance with the potential downside of higher CPU and memory utilization from the Karpenter pod. While the magnitude of this difference varies on a case-by-case basis, when upgrading to Karpenter `0.34.0`+, please note that you may need to increase the resources allocated to the Karpenter controller pods. * Karpenter now adds a default `podSecurityContext` that configures the `fsgroup: 65536` of volumes in the pod. If you are using sidecar containers, you should review if this configuration is compatible for them. You can disable this default `podSecurityContext` through helm by performing `--set podSecurityContext=null` when installing/upgrading the chart. * The `dnsPolicy` for the Karpenter controller pod has been changed back to the Kubernetes cluster default of `ClusterFirst`. Setting our `dnsPolicy` to `Default` (confusingly, this is not the Kubernetes cluster default) caused more confusion for any users running IPv6 clusters with dual-stack nodes or anyone running Karpenter with dependencies on cluster services (like clusters running service meshes). This change may be breaking for any users on Fargate or MNG who were allowing Karpenter to manage their in-cluster DNS service (`core-dns` on most clusters). If you still want the old behavior here, you can change the `dnsPolicy` to point to use `Default` by setting the helm value on install/upgrade with `--set dnsPolicy=Default`. More details on this issue can be found in the following Github issues: [#2186](https://github.com/aws/karpenter-provider-aws/issues/2186) and [#4947](https://github.com/aws/karpenter-provider-aws/issues/4947). * Karpenter now disallows `nodepool.spec.template.spec.resources` to be set. The webhook validation never allowed `nodepool.spec.template.spec.resources`. We are now ensuring that CEL validation also disallows `nodepool.spec.template.spec.resources` to be set. If you were previously setting the resources field on your NodePool, ensure that you remove this field before upgrading to the newest version of Karpenter or else updates to the resource may fail on the new version. -### Upgrading to v0.33.0+ - -[comment]: <> (WHEN CREATING A NEW SECTION OF THE UPGRADE GUIDANCE FOR NEWER VERSIONS, ENSURE THAT YOU COPY THE ALERT SECTION BELOW TO PROPERLY WARN USERS OF THE RISK OF UPGRADING WITHOUT GOING TO v0.32 FIRST) +### Upgrading to `0.33.0`+ {{% alert title="Warning" color="warning" %}} -v0.33.0+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. **Do not** upgrade to v0.33.0+ without first [upgrading to v0.32.x]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. **Do not** upgrade to `0.33.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. {{% /alert %}} * Karpenter no longer supports using the `karpenter.sh/provisioner-name` label in NodePool labels and requirements or in application node selectors, affinities, or topologySpreadConstraints. If you were previously using this label to target applications to specific Provisioners, you should update your applications to use the `karpenter.sh/nodepool` label instead before upgrading. If you upgrade without changing these labels, you may begin to see pod scheduling failures for these applications. * Karpenter now tags `spot-instances-request` with the same tags that it tags instances, volumes, and primary ENIs. This means that you will now need to add `ec2:CreateTags` permission for `spot-instances-request`. You can also further scope your controller policy for the `ec2:RunInstances` action to require that it launches the `spot-instances-request` with these specific tags. You can view an example of scoping these actions in the [Getting Started Guide's default CloudFormation controller policy](https://github.com/aws/karpenter/blob/v0.33.0/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml#L61). * We now recommend that you set the installation namespace for your Karpenter controllers to `kube-system` to denote Karpenter as a critical cluster component. This ensures that requests from the Karpenter controllers are treated with higher priority by assigning them to a different [PriorityLevelConfiguration](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) than generic requests from other namespaces. For more details on API Priority and Fairness, read the [Kubernetes API Priority and Fairness Conceptual Docs](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). Note: Changing the namespace for your Karpenter release will cause the service account namespace to change. If you are using IRSA for authentication with AWS, you will need to change scoping set in the controller's trust policy from `karpenter:karpenter` to `kube-system:karpenter`. -* `v0.33.x` disables mutating and validating webhooks by default in favor of using [Common Expression Language for CRD validation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation). The Common Expression Language Validation Feature [is enabled by default on EKS 1.25](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation-rules). If you are using Kubernetes version >= 1.25, no further action is required. If you are using a Kubernetes version below 1.25, you now need to set `DISABLE_WEBHOOK=false` in your container environment variables or `--set webhook.enabled=true` if using Helm. View the [Webhook Support Deprecated in Favor of CEL Section of the v1beta1 Migration Guide]({{}}). -* `v0.33.x` drops support for passing settings through the `karpenter-global-settings` ConfigMap. You should pass settings through the container environment variables in the Karpenter deployment manifest. View the [Global Settings Section of the v1beta1 Migration Guide]({{}}) for more details. -* `v0.33.x` enables `Drift=true` by default in the `FEATURE_GATES`. If you previously didn't enable the feature gate, Karpenter will now check if there is a difference between the desired state of your nodes declared in your NodePool and the actual state of your nodes. View the [Drift Section of Disruption Conceptual Docs]({{}}) for more details. -* `v0.33.x` drops looking up the `zap-logger-config` through ConfigMap discovery. Instead, Karpenter now expects the logging config to be mounted on the filesystem if you are using this to configure Zap logging. This is not enabled by default, but can be enabled through `--set logConfig.enabled=true` in the helm values. If you are setting any values in the `logConfig` from the `v0.32.x` upgrade, such as `logConfig.logEncoding`, note that you will have to explicitly set `logConfig.enabled=true` alongside it. Also, note that setting the Zap logging config is a deprecated feature in beta and is planned to be dropped at v1. View the [Logging Configuration Section of the v1beta1 Migration Guide]({{}}) for more details. -* `v0.33.x` change the default `LOG_LEVEL` from `debug` to `info` by default. If you are still enabling logging configuration through the `zap-logger-config`, no action is required. -* `v0.33.x` drops support for comma delimited lists on tags for `SubnetSelectorTerm`, `SecurityGroupsSelectorTerm`, and `AMISelectorTerm`. Karpenter now supports multiple terms for each of the selectors which means that we can specify a more explicit OR-based constraint through separate terms rather than a comma-delimited list of values. +* `0.33.0` disables mutating and validating webhooks by default in favor of using [Common Expression Language for CRD validation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation). The Common Expression Language Validation Feature [is enabled by default on EKS 1.25](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation-rules). If you are using Kubernetes version >= 1.25, no further action is required. If you are using a Kubernetes version below 1.25, you now need to set `DISABLE_WEBHOOK=false` in your container environment variables or `--set webhook.enabled=true` if using Helm. View the [Webhook Support Deprecated in Favor of CEL Section of the v1beta1 Migration Guide]({{}}). +* `0.33.0` drops support for passing settings through the `karpenter-global-settings` ConfigMap. You should pass settings through the container environment variables in the Karpenter deployment manifest. View the [Global Settings Section of the v1beta1 Migration Guide]({{}}) for more details. +* `0.33.0` enables `Drift=true` by default in the `FEATURE_GATES`. If you previously didn't enable the feature gate, Karpenter will now check if there is a difference between the desired state of your nodes declared in your NodePool and the actual state of your nodes. View the [Drift Section of Disruption Conceptual Docs]({{}}) for more details. +* `0.33.0` drops looking up the `zap-logger-config` through ConfigMap discovery. Instead, Karpenter now expects the logging config to be mounted on the filesystem if you are using this to configure Zap logging. This is not enabled by default, but can be enabled through `--set logConfig.enabled=true` in the Helm values. If you are setting any values in the `logConfig` from the `0.32.x` upgrade, such as `logConfig.logEncoding`, note that you will have to explicitly set `logConfig.enabled=true` alongside it. Also, note that setting the Zap logging config is a deprecated feature in beta and is planned to be dropped at v1. View the [Logging Configuration Section of the v1beta1 Migration Guide]({{}}) for more details. +* `0.33.0` change the default `LOG_LEVEL` from `debug` to `info` by default. If you are still enabling logging configuration through the `zap-logger-config`, no action is required. +* `0.33.0` drops support for comma delimited lists on tags for `SubnetSelectorTerm`, `SecurityGroupsSelectorTerm`, and `AMISelectorTerm`. Karpenter now supports multiple terms for each of the selectors which means that we can specify a more explicit OR-based constraint through separate terms rather than a comma-delimited list of values. -### Upgrading to v0.32.0+ +### Upgrading to `0.32.0`+ {{% alert title="Warning" color="warning" %}} -Karpenter v0.32.0 introduces v1beta1 APIs, including _significant_ changes to the API and installation procedures for the Karpenter controllers. **Do not** upgrade to v0.32.0+ without referencing the [v1beta1 Migration Upgrade Procedure]({{}}). +Karpenter `0.32.0` introduces v1beta1 APIs, including _significant_ changes to the API and installation procedures for the Karpenter controllers. **Do not** upgrade to `0.32.0`+ without referencing the [v1beta1 Migration Upgrade Procedure]({{}}). This version includes **dual support** for both alpha and beta APIs to ensure that you can slowly migrate your existing Provisioner, AWSNodeTemplate, and Machine alpha APIs to the newer NodePool, EC2NodeClass, and NodeClaim beta APIs. -Note that if you are rolling back after upgrading to v0.32.0, note that v0.31.4 is the only version that supports handling rollback after you have deployed the v1beta1 APIs to your cluster. +Note that if you are rolling back after upgrading to `0.32.0`, note that __only__ versions `0.31.4` support handling rollback after you have deployed the v1beta1 APIs to your cluster. {{% /alert %}} +* Karpenter now uses `settings.InterruptionQueue` instead of `settings.aws.InterruptionQueueName` in its helm chart. The CLI argument also changed to `--interruption-queue`. * Karpenter now serves the webhook prometheus metrics server on port `8001`. If this port is already in-use on the pod or you are running in `hostNetworking` mode, you may need to change this port value. You can configure this port value through the `WEBHOOK_METRICS_PORT` environment variable or the `webhook.metrics.port` value if installing via Helm. * Karpenter now exposes the ability to disable webhooks through the `webhook.enabled=false` value. This value will disable the webhook server and will prevent any permissions, mutating or validating webhook configurations from being deployed to the cluster. * Karpenter now moves all logging configuration for the Zap logger into the `logConfig` values block. Configuring Karpenter logging with this mechanism _is_ deprecated and will be dropped at v1. Karpenter now only surfaces logLevel through the `logLevel` helm value. If you need more advanced configuration due to log parsing constraints, we recommend configuring your log parser to handle Karpenter's Zap JSON logging. -* The default log encoding changed from `console` to `json`. If you were previously not setting the type of log encoding, this default will change with the helm chart. If you were setting the value through `logEncoding`, this value will continue to work until v0.33.x but it is deprecated in favor of `logConfig.logEncoding` +* The default log encoding changed from `console` to `json`. If you were previously not setting the type of log encoding, this default will change with the Helm chart. If you were setting the value through `logEncoding`, this value will continue to work until `0.33.x` but it is deprecated in favor of `logConfig.logEncoding` * Karpenter now uses the `karpenter.sh/disruption:NoSchedule=disrupting` taint instead of the upstream `node.kubernetes.io/unschedulable` taint for nodes spawned with a NodePool to prevent pods from scheduling to nodes being disrupted. Pods that previously tolerated the `node.kubernetes.io/unschedulable` taint that previously weren't evicted during termination will now be evicted. This most notably affects DaemonSets, which have the `node.kubernetes.io/unschedulable` toleration by default, where Karpenter will now remove these pods during termination. If you want your specific pods to not be evicted when nodes are scaled down, you should add a toleration to the pods with the following: `Key=karpenter.sh/disruption, Effect=NoSchedule, Operator=Equals, Values=disrupting`. * Note: Karpenter will continue to use the old `node.kubernetes.io/unschedulable` taint for nodes spawned with a Provisioner. -### Upgrading to v0.31.0+ +### Upgrading to `0.31.0`+ * Karpenter moved its `securityContext` constraints from pod-wide to only applying to the Karpenter container exclusively. If you were previously relying on the pod-wide `securityContext` for your sidecar containers, you will now need to set these values explicitly in your sidecar container configuration. -### Upgrading to v0.30.0+ +### Upgrading to `0.30.0`+ * Karpenter will now [statically drift]({{}}) on both Provisioner and AWSNodeTemplate Fields. For Provisioner Static Drift, the `karpenter.sh/provisioner-hash` annotation must be present on both the Provisioner and Machine. For AWSNodeTemplate drift, the `karpenter.k8s.aws/nodetemplate-hash` annotation must be present on the AWSNodeTemplate and Machine. Karpenter will not add these annotations to pre-existing nodes, so each of these nodes will need to be recycled one time for the annotations to be added. * Karpenter will now fail validation on AWSNodeTemplates and Provisioner `spec.provider` that have `amiSelectors`, `subnetSelectors`, or `securityGroupSelectors` set with a combination of id selectors (`aws-ids`, `aws::ids`) and other selectors. -* Karpenter now statically sets the `securityContext` at both the pod and container-levels and doesn't allow override values to be passed through the helm chart. This change was made to adhere to [Restricted Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted), which follows pod hardening best practices. +* Karpenter now statically sets the `securityContext` at both the pod and container-levels and doesn't allow override values to be passed through the Helm chart. This change was made to adhere to [Restricted Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted), which follows pod hardening best practices. {{% alert title="Note" color="primary" %}} If you have sidecar containers configured to run alongside Karpenter that cannot tolerate the [pod-wide `securityContext` constraints](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/deployment.yaml#L40), you will need to specify overrides to the sidecar `securityContext` in your deployment. {{% /alert %}} -### Upgrading to v0.29.0+ +### Upgrading to `0.29.0`+ {{% alert title="Warning" color="warning" %}} -Karpenter `v0.29.1` contains a [file descriptor and memory leak bug](https://github.com/aws/karpenter/issues/4296) that leads to Karpenter getting OOMKilled and restarting at the point that it hits its memory or file descriptor limit. Karpenter `>v0.29.2` fixes this leak. +Karpenter `0.29.1` contains a [file descriptor and memory leak bug](https://github.com/aws/karpenter/issues/4296) that leads to Karpenter getting OOMKilled and restarting at the point that it hits its memory or file descriptor limit. Karpenter `0.29.2`+ fixes this leak. {{% /alert %}} -* Karpenter has changed the default metrics service port from 8080 to 8000 and the default webhook service port from 443 to 8443. In `v0.28.0`, the Karpenter pod port was changed to 8000, but referenced the service by name, allowing users to scrape the service at port 8080 for metrics. `v0.29.0` aligns the two ports so that service and pod metrics ports are the same. These ports are set by the `controller.metrics.port` and `webhook.port` helm chart values, so if you have previously set these to non-default values, you may need to update your Prometheus scraper to match these new values. +* Karpenter has changed the default metrics service port from 8080 to 8000 and the default webhook service port from 443 to 8443. In `0.28.0`, the Karpenter pod port was changed to 8000, but referenced the service by name, allowing users to scrape the service at port 8080 for metrics. `0.29.0` aligns the two ports so that service and pod metrics ports are the same. These ports are set by the `controller.metrics.port` and `webhook.port` Helm chart values, so if you have previously set these to non-default values, you may need to update your Prometheus scraper to match these new values. * Karpenter will now reconcile nodes that are drifted due to their Security Groups or their Subnets. If your AWSNodeTemplate's Security Groups differ from the Security Groups used for an instance, Karpenter will consider it drifted. If the Subnet used by an instance is not contained in the allowed list of Subnets for an AWSNodeTemplate, Karpenter will also consider it drifted. * Since Karpenter uses tags for discovery of Subnets and SecurityGroups, check the [Threat Model]({{}}) to see how to manage this IAM Permission. -### Upgrading to v0.28.0+ +### Upgrading to `0.28.0`+ {{% alert title="Warning" color="warning" %}} -Karpenter `v0.28.0` is incompatible with Kubernetes version 1.26+, which can result in additional node scale outs when using `--cloudprovider=external`, which is the default for the EKS Optimized AMI. See: https://github.com/aws/karpenter-core/pull/375. Karpenter `>v0.28.1` fixes this issue and is compatible with Kubernetes version 1.26+. +Karpenter `0.28.0` is incompatible with Kubernetes version 1.26+, which can result in additional node scale outs when using `--cloudprovider=external`, which is the default for the EKS Optimized AMI. See: https://github.com/aws/karpenter-core/pull/375. Karpenter `0.28.1`+ fixes this issue and is compatible with Kubernetes version 1.26+. {{% /alert %}} -* The `extraObjects` value is now removed from the Helm chart. Having this value in the chart proved to not work in the majority of Karpenter installs and often led to anti-patterns, where the Karpenter resources installed to manage Karpenter's capacity were directly tied to the install of the Karpenter controller deployments. The Karpenter team recommends that, if you want to install Karpenter manifests alongside the Karpenter helm chart, to do so by creating a separate chart for the manifests, creating a dependency on the controller chart. +* The `extraObjects` value is now removed from the Helm chart. Having this value in the chart proved to not work in the majority of Karpenter installs and often led to anti-patterns, where the Karpenter resources installed to manage Karpenter's capacity were directly tied to the install of the Karpenter controller deployments. The Karpenter team recommends that, if you want to install Karpenter manifests alongside the Karpenter Helm chart, to do so by creating a separate chart for the manifests, creating a dependency on the controller chart. * The `aws.nodeNameConvention` setting is now removed from the [`karpenter-global-settings`]({{}}) ConfigMap. Because Karpenter is now driving its orchestration of capacity through Machines, it no longer needs to know the node name, making this setting obsolete. Karpenter ignores configuration that it doesn't recognize in the [`karpenter-global-settings`]({{}}) ConfigMap, so leaving the `aws.nodeNameConvention` in the ConfigMap will simply cause this setting to be ignored. * Karpenter now defines a set of "restricted tags" which can't be overridden with custom tagging in the AWSNodeTemplate or in the [`karpenter-global-settings`]({{}}) ConfigMap. If you are currently using any of these tag overrides when tagging your instances, webhook validation will now fail. These tags include: @@ -133,7 +165,7 @@ Karpenter `v0.28.0` is incompatible with Kubernetes version 1.26+, which can res * `karpenter_nodes_terminated`: Use `karpenter_machines_terminated` if you are interested in the reason why a Karpenter machine was deleted. `karpenter_nodes_terminated` now only tracks the count of terminated nodes without any additional labels. * `karpenter_nodes_created`: Use `karpenter_machines_created` if you are interested in the reason why a Karpenter machine was created. `karpenter_nodes_created` now only tracks the count of created nodes without any additional labels. * `karpenter_deprovisioning_replacement_node_initialized_seconds`: This metric has been replaced in favor of `karpenter_deprovisioning_replacement_machine_initialized_seconds`. -* `v0.28.0` introduces the Machine CustomResource into the `karpenter.sh` API Group and requires this CustomResourceDefinition to run properly. Karpenter now orchestrates its CloudProvider capacity through these in-cluster Machine CustomResources. When performing a scheduling decision, Karpenter will create a Machine, resulting in launching CloudProvider capacity. The kubelet running on the new capacity will then register the node to the cluster shortly after launch. +* `0.28.0` introduces the Machine CustomResource into the `karpenter.sh` API Group and requires this CustomResourceDefinition to run properly. Karpenter now orchestrates its CloudProvider capacity through these in-cluster Machine CustomResources. When performing a scheduling decision, Karpenter will create a Machine, resulting in launching CloudProvider capacity. The kubelet running on the new capacity will then register the node to the cluster shortly after launch. * If you are using Helm to upgrade between versions of Karpenter, note that [Helm does not automate the process of upgrading or install the new CRDs into your cluster](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations). To install or upgrade the existing CRDs, follow the guidance under the [Custom Resource Definition (CRD) Upgrades]({{< relref "#custom-resource-definition-crd-upgrades" >}}) section of the upgrade guide. * Karpenter will hydrate Machines on startup for existing capacity managed by Karpenter into the cluster. Existing capacity launched by an older version of Karpenter is discovered by finding CloudProvider capacity with the `karpenter.sh/provisioner-name` tag or the `karpenter.sh/provisioner-name` label on nodes. * The metrics port for the Karpenter deployment was changed from 8080 to 8000. Users who scrape the pod directly for metrics rather than the service will need to adjust the commands they use to reference port 8000. Any users who scrape metrics from the service should be unaffected. @@ -149,19 +181,21 @@ Because Karpenter takes this dependency, any user that has the ability to Create {{% /alert %}} {{% alert title="Rolling Back" color="warning" %}} -If, after upgrading to `v0.28.0+`, a rollback to an older version of Karpenter needs to be performed, Karpenter will continue to function normally, though you will still have the Machine CustomResources on your cluster. You will need to manually delete the Machines and patch out the finalizers to fully complete the rollback. +If, after upgrading to `0.28.0`+, a rollback to an older version of Karpenter needs to be performed, Karpenter will continue to function normally, though you will still have the Machine CustomResources on your cluster. You will need to manually delete the Machines and patch out the finalizers to fully complete the rollback. -Karpenter marks CloudProvider capacity as "managed by" a Machine using the `karpenter-sh/managed-by` tag on the CloudProvider machine. It uses this tag to ensure that the Machine CustomResources in the cluster match the CloudProvider capacity managed by Karpenter. If these states don't match, Karpenter will garbage collect the capacity. Because of this, if performing an upgrade, followed by a rollback, followed by another upgrade to `v0.28.0+`, ensure you remove the `karpenter.sh/managed-by` tags from existing capacity; otherwise, Karpenter will deprovision the capacity without a Machine CR counterpart. +Karpenter marks CloudProvider capacity as "managed by" a Machine using the `karpenter-sh/managed-by` tag on the CloudProvider machine. It uses this tag to ensure that the Machine CustomResources in the cluster match the CloudProvider capacity managed by Karpenter. If these states don't match, Karpenter will garbage collect the capacity. Because of this, if performing an upgrade, followed by a rollback, followed by another upgrade to `0.28.0`+, ensure you remove the `karpenter.sh/managed-by` tags from existing capacity; otherwise, Karpenter will deprovision the capacity without a Machine CR counterpart. {{% /alert %}} -### Upgrading to v0.27.3+ -* The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `v0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. +### Upgrading to `0.27.3`+ + +* The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. ```bash kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh ``` -### Upgrading to v0.27.0+ +### Upgrading to `0.27.0`+ + * The Karpenter controller pods now deploy with `kubernetes.io/hostname` self anti-affinity by default. If you are running Karpenter in HA (high-availability) mode and you do not have enough nodes to match the number of pod replicas you are deploying with, you will need to scale-out your nodes for Karpenter. * The following controller metrics changed and moved under the `controller_runtime` metrics namespace: * `karpenter_metricscraper_...` @@ -178,27 +212,33 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh * `provisioner-state` -> `provisioner_state` * The `karpenter_allocation_controller_scheduling_duration_seconds` metric name changed to `karpenter_provisioner_scheduling_duration_seconds` -### Upgrading to v0.26.0+ +### Upgrading to `0.26.0`+ + * The `karpenter.sh/do-not-evict` annotation no longer blocks node termination when running `kubectl delete node`. This annotation on pods will only block automatic deprovisioning that is considered "voluntary," that is, disruptions that can be avoided. Disruptions that Karpenter deems as "involuntary" and will ignore the `karpenter.sh/do-not-evict` annotation include spot interruption and manual deletion of the node. See [Disabling Deprovisioning]({{}}) for more details. -* Default resources `requests` and `limits` are removed from the Karpenter's controller deployment through the Helm chart. If you have not set custom resource `requests` or `limits` in your helm values and are using Karpenter's defaults, you will now need to set these values in your helm chart deployment. -* The `controller.image` value in the helm chart has been broken out to a map consisting of `controller.image.repository`, `controller.image.tag`, and `controller.image.digest`. If manually overriding the `controller.image`, you will need to update your values to the new design. +* Default resources `requests` and `limits` are removed from the Karpenter's controller deployment through the Helm chart. If you have not set custom resource `requests` or `limits` in your Helm values and are using Karpenter's defaults, you will now need to set these values in your Helm chart deployment. +* The `controller.image` value in the Helm chart has been broken out to a map consisting of `controller.image.repository`, `controller.image.tag`, and `controller.image.digest`. If manually overriding the `controller.image`, you will need to update your values to the new design. + +### Upgrading to `0.25.0`+ -### Upgrading to v0.25.0+ * Cluster Endpoint can now be automatically discovered. If you are using Amazon Elastic Kubernetes Service (EKS), you can now omit the `clusterEndpoint` field in your configuration. In order to allow the resolving, you have to add the permission `eks:DescribeCluster` to the Karpenter Controller IAM role. -### Upgrading to v0.24.0+ +### Upgrading to `0.24.0`+ + * Settings are no longer updated dynamically while Karpenter is running. If you manually make a change to the [`karpenter-global-settings`]({{}}) ConfigMap, you will need to reload the containers by restarting the deployment with `kubectl rollout restart -n karpenter deploy/karpenter` * Karpenter no longer filters out instance types internally. Previously, `g2` (not supported by the NVIDIA device plugin) and FPGA instance types were filtered. The only way to filter instance types now is to set requirements on your provisioner or pods using well-known node labels described [here]({{}}). If you are currently using overly broad requirements that allows all of the `g` instance-category, you will want to tighten the requirement, or add an instance-generation requirement. * `aws.tags` in [`karpenter-global-settings`]({{}}) ConfigMap is now a top-level field and expects the value associated with this key to be a JSON object of string to string. This is change from previous versions where keys were given implicitly by providing the key-value pair `aws.tags.: value` in the ConfigMap. -### Upgrading to v0.22.0+ +### Upgrading to `0.22.0`+ + * Do not upgrade to this version unless you are on Kubernetes >= v1.21. Karpenter no longer supports Kubernetes v1.20, but now supports Kubernetes v1.25. This change is due to the v1 PDB API, which was introduced in K8s v1.20 and subsequent removal of the v1beta1 API in K8s v1.25. -### Upgrading to v0.20.0+ -* Prior to v0.20.0, Karpenter would prioritize certain instance type categories absent of any requirements in the Provisioner. v0.20.0+ removes prioritizing these instance type categories ("m", "c", "r", "a", "t", "i") in code. Bare Metal and GPU instance types are still deprioritized and only used if no other instance types are compatible with the node requirements. Since Karpenter does not prioritize any instance types, if you do not want exotic instance types and are not using the runtime Provisioner defaults, you will need to specify this in the Provisioner. +### Upgrading to `0.20.0`+ + +* Prior to `0.20.0`, Karpenter would prioritize certain instance type categories absent of any requirements in the Provisioner. `0.20.0`+ removes prioritizing these instance type categories ("m", "c", "r", "a", "t", "i") in code. Bare Metal and GPU instance types are still deprioritized and only used if no other instance types are compatible with the node requirements. Since Karpenter does not prioritize any instance types, if you do not want exotic instance types and are not using the runtime Provisioner defaults, you will need to specify this in the Provisioner. + +### Upgrading to `0.19.0`+ -### Upgrading to v0.19.0+ -* The karpenter webhook and controller containers are combined into a single binary, which requires changes to the helm chart. If your Karpenter installation (helm or otherwise) currently customizes the karpenter webhook, your deployment tooling may require minor changes. +* The karpenter webhook and controller containers are combined into a single binary, which requires changes to the Helm chart. If your Karpenter installation (Helm or otherwise) currently customizes the karpenter webhook, your deployment tooling may require minor changes. * Karpenter now supports native interruption handling. If you were previously using Node Termination Handler for spot interruption handling and health events, you will need to remove the component from your cluster before enabling `aws.interruptionQueueName`. For more details on Karpenter's interruption handling, see the [Interruption Handling Docs]({{< ref "../concepts/disruption/#interruption" >}}). * Instance category defaults are now explicitly persisted in the Provisioner, rather than handled implicitly in memory. By default, Provisioners will limit instance category to c,m,r. If any instance type constraints are applied, it will override this default. If you have created Provisioners in the past with unconstrained instance type, family, or category, Karpenter will now more flexibly use instance types than before. If you would like to apply these constraints, they must be included in the Provisioner CRD. * Karpenter CRD raw YAML URLs have migrated from `https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.19.3/charts/karpenter/crds/...` to `https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.19.3/pkg/apis/crds/...`. If you reference static Karpenter CRDs or rely on `kubectl replace -f` to apply these CRDs from their remote location, you will need to migrate to the new location. @@ -214,38 +254,44 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh * `AWS_NODE_NAME_CONVENTION` -> `settings.aws.nodeNameConvention` * `VM_MEMORY_OVERHEAD` -> `settings.aws.vmMemoryOverheadPercent` -### Upgrading to v0.18.0+ -* v0.18.0 removes the `karpenter_consolidation_nodes_created` and `karpenter_consolidation_nodes_terminated` prometheus metrics in favor of the more generic `karpenter_nodes_created` and `karpenter_nodes_terminated` metrics. You can still see nodes created and terminated by consolidation by checking the `reason` label on the metrics. Check out all the metrics published by Karpenter [here]({{}}). +### Upgrading to `0.18.0`+ + +* `0.18.0` removes the `karpenter_consolidation_nodes_created` and `karpenter_consolidation_nodes_terminated` prometheus metrics in favor of the more generic `karpenter_nodes_created` and `karpenter_nodes_terminated` metrics. You can still see nodes created and terminated by consolidation by checking the `reason` label on the metrics. Check out all the metrics published by Karpenter [here]({{}}). + +### Upgrading to `0.17.0`+ -### Upgrading to v0.17.0+ Karpenter's Helm chart package is now stored in [Karpenter's OCI (Open Container Initiative) registry](https://gallery.ecr.aws/karpenter/karpenter). The Helm CLI supports the new format since [v3.8.0+](https://helm.sh/docs/topics/registries/). -With this change [charts.karpenter.sh](https://charts.karpenter.sh/) is no longer updated but preserved to allow using older Karpenter versions. For examples on working with the Karpenter helm charts look at [Install Karpenter Helm Chart]({{< ref "../getting-started/getting-started-with-karpenter/#install-karpenter-helm-chart" >}}). +With this change [charts.karpenter.sh](https://charts.karpenter.sh/) is no longer updated but preserved to allow using older Karpenter versions. For examples on working with the Karpenter Helm charts look at [Install Karpenter Helm Chart]({{< ref "../getting-started/getting-started-with-karpenter/#install-karpenter-helm-chart" >}}). Users who have scripted the installation or upgrading of Karpenter need to adjust their scripts with the following changes: -1. There is no longer a need to add the Karpenter helm repo to helm -2. The full URL of the Helm chart needs to be present when using the helm commands -3. If you were not prepending a `v` to the version (i.e. `0.17.0`), you will need to do so with the OCI chart, `v0.17.0`. +1. There is no longer a need to add the Karpenter Helm repo with `helm repo add` +2. The full URL of the Helm chart needs to be present when using the `helm` CLI +3. If you were not prepending a `v` to the version (i.e. `0.17.0`), you will need to do so with the OCI chart (i.e `v0.17.0`). -### Upgrading to v0.16.2+ -* v0.16.2 adds new kubeletConfiguration fields to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: +### Upgrading to `0.16.2`+ + +* `0.16.2` adds new kubeletConfiguration fields to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: ```bash kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.16.2/charts/karpenter/crds/karpenter.sh_provisioners.yaml ``` -### Upgrading to v0.16.0+ -* v0.16.0 adds a new weight field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: +### Upgrading to `0.16.0`+ + +* `0.16.0` adds a new weight field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: ```bash kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.16.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml ``` -### Upgrading to v0.15.0+ -* v0.15.0 adds a new consolidation field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: +### Upgrading to `0.15.0`+ + +* `0.15.0` adds a new consolidation field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: ```bash kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.15.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml ``` -### Upgrading to v0.14.0+ -* v0.14.0 adds new fields to the `provisioners.karpenter.sh` v1alpha5 and `awsnodetemplates.karpenter.k8s.aws` v1alpha1 CRDs. The CRDs will need to be updated to use the new parameters: +### Upgrading to `0.14.0`+ + +* `0.14.0` adds new fields to the `provisioners.karpenter.sh` v1alpha5 and `awsnodetemplates.karpenter.k8s.aws` v1alpha1 CRDs. The CRDs will need to be updated to use the new parameters: ```bash kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.14.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml @@ -253,7 +299,7 @@ kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/ kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.14.0/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml ``` -* v0.14.0 changes the way Karpenter discovers its dynamically generated AWS launch templates to use a tag rather than a Name scheme. The previous name scheme was `Karpenter-${CLUSTER_NAME}-*` which could collide with user created launch templates that Karpenter should not manage. The new scheme uses a tag on the launch template `karpenter.k8s.aws/cluster: ${CLUSTER_NAME}`. As a result, Karpenter will not clean-up dynamically generated launch templates using the old name scheme. You can manually clean these up with the following commands: +* `0.14.0` changes the way Karpenter discovers its dynamically generated AWS launch templates to use a tag rather than a Name scheme. The previous name scheme was `Karpenter-${CLUSTER_NAME}-*` which could collide with user created launch templates that Karpenter should not manage. The new scheme uses a tag on the launch template `karpenter.k8s.aws/cluster: ${CLUSTER_NAME}`. As a result, Karpenter will not clean-up dynamically generated launch templates using the old name scheme. You can manually clean these up with the following commands: ```bash ## Find launch templates that match the naming pattern and you do not want to keep @@ -263,52 +309,54 @@ aws ec2 describe-launch-templates --filters="Name=launch-template-name,Values=Ka aws ec2 delete-launch-template --launch-template-id ``` -* v0.14.0 introduces additional instance type filtering if there are no `node.kubernetes.io/instance-type` or `karpenter.k8s.aws/instance-family` or `karpenter.k8s.aws/instance-category` requirements that restrict instance types specified on the provisioner. This prevents Karpenter from launching bare metal and some older non-current generation instance types unless the provisioner has been explicitly configured to allow them. If you specify an instance type or family requirement that supplies a list of instance-types or families, that list will be used regardless of filtering. The filtering can also be completely eliminated by adding an `Exists` requirement for instance type or family. +* `0.14.0` introduces additional instance type filtering if there are no `node.kubernetes.io/instance-type` or `karpenter.k8s.aws/instance-family` or `karpenter.k8s.aws/instance-category` requirements that restrict instance types specified on the provisioner. This prevents Karpenter from launching bare metal and some older non-current generation instance types unless the provisioner has been explicitly configured to allow them. If you specify an instance type or family requirement that supplies a list of instance-types or families, that list will be used regardless of filtering. The filtering can also be completely eliminated by adding an `Exists` requirement for instance type or family. ```yaml - key: node.kubernetes.io/instance-type operator: Exists ``` -* v0.14.0 introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here]({{}}) to get started -* v0.14.0 adds an an additional default toleration (CriticalAddonOnly=Exists) to the Karpenter helm chart. This may cause Karpenter to run on nodes with that use this Taint which previously would not have been schedulable. This can be overridden by using `--set tolerations[0]=null`. +* `0.14.0` introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here]({{}}) to get started +* `0.14.0` adds an an additional default toleration (CriticalAddonOnly=Exists) to the Karpenter Helm chart. This may cause Karpenter to run on nodes with that use this Taint which previously would not have been schedulable. This can be overridden by using `--set tolerations[0]=null`. -* v0.14.0 deprecates the `AWS_ENI_LIMITED_POD_DENSITY` environment variable in-favor of specifying `spec.kubeletConfiguration.maxPods` on the Provisioner. `AWS_ENI_LIMITED_POD_DENSITY` will continue to work when `maxPods` is not set on the Provisioner. If `maxPods` is set, it will override `AWS_ENI_LIMITED_POD_DENSITY` on that specific Provisioner. +* `0.14.0` deprecates the `AWS_ENI_LIMITED_POD_DENSITY` environment variable in-favor of specifying `spec.kubeletConfiguration.maxPods` on the Provisioner. `AWS_ENI_LIMITED_POD_DENSITY` will continue to work when `maxPods` is not set on the Provisioner. If `maxPods` is set, it will override `AWS_ENI_LIMITED_POD_DENSITY` on that specific Provisioner. -### Upgrading to v0.13.0+ -* v0.13.0 introduces a new CRD named `AWSNodeTemplate` which can be used to specify AWS Cloud Provider parameters. Everything that was previously specified under `spec.provider` in the Provisioner resource, can now be specified in the spec of the new resource. The use of `spec.provider` is deprecated but will continue to function to maintain backwards compatibility for the current API version (v1alpha5) of the Provisioner resource. v0.13.0 also introduces support for custom user data that doesn't require the use of a custom launch template. The user data can be specified in-line in the AWSNodeTemplate resource. +### Upgrading to `0.13.0`+ - If you are upgrading from v0.10.1 - v0.11.1, a new CRD `awsnodetemplate` was added. In v0.12.0, this crd was renamed to `awsnodetemplates`. Since helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: +* `0.13.0` introduces a new CRD named `AWSNodeTemplate` which can be used to specify AWS Cloud Provider parameters. Everything that was previously specified under `spec.provider` in the Provisioner resource, can now be specified in the spec of the new resource. The use of `spec.provider` is deprecated but will continue to function to maintain backwards compatibility for the current API version (v1alpha5) of the Provisioner resource. `0.13.0` also introduces support for custom user data that doesn't require the use of a custom launch template. The user data can be specified in-line in the AWSNodeTemplate resource. + + If you are upgrading from `0.10.1` - `0.11.1`, a new CRD `awsnodetemplate` was added. In `0.12.0`, this crd was renamed to `awsnodetemplates`. Since Helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: 1. Make sure any `awsnodetemplate` manifests are saved somewhere so that they can be reapplied to the cluster. 2. `kubectl delete crd awsnodetemplate` 3. `kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.13.2/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml` - 4. Perform the Karpenter upgrade to v0.13.x, which will install the new `awsnodetemplates` CRD. + 4. Perform the Karpenter upgrade to `0.13.0`+, which will install the new `awsnodetemplates` CRD. 5. Reapply the `awsnodetemplate` manifests you saved from step 1, if applicable. -* v0.13.0 also adds EC2/spot price fetching to Karpenter to allow making more accurate decisions regarding node deployments. Our [getting started guide]({{< ref "../getting-started/getting-started-with-karpenter" >}}) documents this, but if you are upgrading Karpenter you will need to modify your Karpenter controller policy to add the `pricing:GetProducts` and `ec2:DescribeSpotPriceHistory` permissions. +* `0.13.0` also adds EC2/spot price fetching to Karpenter to allow making more accurate decisions regarding node deployments. Our [getting started guide]({{< ref "../getting-started/getting-started-with-karpenter" >}}) documents this, but if you are upgrading Karpenter you will need to modify your Karpenter controller policy to add the `pricing:GetProducts` and `ec2:DescribeSpotPriceHistory` permissions. + +### Upgrading to `0.12.0`+ -### Upgrading to v0.12.0+ -* v0.12.0 adds an OwnerReference to each Node created by a provisioner. Previously, deleting a provisioner would orphan nodes. Now, deleting a provisioner will cause Kubernetes [cascading delete](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#cascading-deletion) logic to gracefully terminate the nodes using the Karpenter node finalizer. You may still orphan nodes by removing the owner reference. -* If you are upgrading from v0.10.1 - v0.11.1, a new CRD `awsnodetemplate` was added. In v0.12.0, this crd was renamed to `awsnodetemplates`. Since helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: +* `0.12.0` adds an OwnerReference to each Node created by a provisioner. Previously, deleting a provisioner would orphan nodes. Now, deleting a provisioner will cause Kubernetes [cascading delete](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#cascading-deletion) logic to gracefully terminate the nodes using the Karpenter node finalizer. You may still orphan nodes by removing the owner reference. +* If you are upgrading from `0.10.1` - `0.11.1`, a new CRD `awsnodetemplate` was added. In `0.12.0`, this crd was renamed to `awsnodetemplates`. Since Helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: 1. Make sure any `awsnodetemplate` manifests are saved somewhere so that they can be reapplied to the cluster. 2. `kubectl delete crd awsnodetemplate` 3. `kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.12.1/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml` - 4. Perform the Karpenter upgrade to v0.12.x, which will install the new `awsnodetemplates` CRD. + 4. Perform the Karpenter upgrade to `0.12.0`+, which will install the new `awsnodetemplates` CRD. 5. Reapply the `awsnodetemplate` manifests you saved from step 1, if applicable. -### Upgrading to v0.11.0+ +### Upgrading to `0.11.0`+ -v0.11.0 changes the way that the `vpc.amazonaws.com/pod-eni` resource is reported. Instead of being reported for all nodes that could support the resources regardless of if the cluster is configured to support it, it is now controlled by a command line flag or environment variable. The parameter defaults to false and must be set if your cluster uses [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html). This can be enabled by setting the environment variable `AWS_ENABLE_POD_ENI` to true via the helm value `controller.env`. +`0.11.0` changes the way that the `vpc.amazonaws.com/pod-eni` resource is reported. Instead of being reported for all nodes that could support the resources regardless of if the cluster is configured to support it, it is now controlled by a command line flag or environment variable. The parameter defaults to false and must be set if your cluster uses [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html). This can be enabled by setting the environment variable `AWS_ENABLE_POD_ENI` to true via the helm value `controller.env`. Other extended resources must be registered on nodes by their respective device plugins which are typically installed as DaemonSets (e.g. the `nvidia.com/gpu` resource will be registered by the [NVIDIA device plugin](https://github.com/NVIDIA/k8s-device-plugin). Previously, Karpenter would register these resources on nodes at creation and they would be zeroed out by `kubelet` at startup. By allowing the device plugins to register the resources, pods will not bind to the nodes before any device plugin initialization has occurred. -v0.11.0 adds a `providerRef` field in the Provisioner CRD. To use this new field you will need to replace the Provisioner CRD manually: +`0.11.0` adds a `providerRef` field in the Provisioner CRD. To use this new field you will need to replace the Provisioner CRD manually: ```shell kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.11.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml ``` -### Upgrading to v0.10.0+ +### Upgrading to `0.10.0`+ -v0.10.0 adds a new field, `startupTaints` to the provisioner spec. Standard Helm upgrades [do not upgrade CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations) so the field will not be available unless the CRD is manually updated. This can be performed prior to the standard upgrade by applying the new CRD manually: +`0.10.0` adds a new field, `startupTaints` to the provisioner spec. Standard Helm upgrades [do not upgrade CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations) so the field will not be available unless the CRD is manually updated. This can be performed prior to the standard upgrade by applying the new CRD manually: ```shell kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.10.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml @@ -316,7 +364,7 @@ kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/ 📝 If you don't perform this manual CRD update, Karpenter will work correctly except for rejecting the creation/update of provisioners that use `startupTaints`. -### Upgrading to v0.6.2+ +### Upgrading to `0.6.2`+ If using Helm, the variable names have changed for the cluster's name and endpoint. You may need to update any configuration that sets the old variable names. diff --git a/website/hugo.yaml b/website/hugo.yaml index a58124bf6053..80c2b0830b97 100644 --- a/website/hugo.yaml +++ b/website/hugo.yaml @@ -76,12 +76,12 @@ params: url: "https://slack.k8s.io/" icon: fab fa-slack desc: "Chat with us on Slack in the #aws-provider channel" - latest_release_version: 0.36.0 - latest_k8s_version: 1.29 + latest_release_version: "0.37.0" + latest_k8s_version: "1.30" versions: + - v0.37 - v0.36 - v0.35 - - v0.34 - v0.32 - preview menu: diff --git a/website/package-lock.json b/website/package-lock.json index e1522aed9821..adca4cb6f913 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -333,12 +333,12 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -867,9 +867,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "dependencies": { "to-regex-range": "^5.0.1" diff --git a/website/static/_redirects b/website/static/_redirects index 01b76800454c..d711f6a139c3 100644 --- a/website/static/_redirects +++ b/website/static/_redirects @@ -1,11 +1,3 @@ -# Redirects from what the browser requests to what we serve -/v0.28.0/* /v0.28/:splat -/v0.27.5/* /v0.27/:splat -/v0.26.1/* /v0.26/:splat -/v0.26/getting-started/getting-started-with-karpenter/* /v0.26/getting-started/getting-started-with-eksctl/:splat -/v0.27/getting-started/getting-started-with-eksctl/* /v0.27/getting-started/getting-started-with-karpenter/:splat -/v0.28/getting-started/getting-started-with-eksctl/* /v0.28/getting-started/getting-started-with-karpenter/:splat - # Redirect all preview documentation to the new routes /preview/concepts/provisioners /preview/concepts/nodepools /preview/concepts/node-templates /preview/concepts/nodeclasses @@ -26,26 +18,6 @@ /docs/concepts/settings /docs/reference/settings /docs/concepts/threat-model /docs/reference/threat-model -# Redirect all v0.30 documentation from the new routes to the old routes -/v0.30/concepts/nodepools /v0.30/concepts/provisioners -/v0.30/concepts/nodeclasses /v0.30/concepts/node-templates -/v0.30/concepts/disruption /v0.30/concepts/deprovisioning -/v0.30/upgrading/upgrade-guide /v0.30/upgrade-guide -/v0.30/reference/instance-types /v0.30/concepts/instance-types -/v0.30/reference/metrics /v0.30/concepts/metrics -/v0.30/reference/settings /v0.30/concepts/settings -/v0.30/reference/threat-model /v0.30/concepts/threat-model - -# Redirect all v0.31 documentation from the new routes to the old routes -/v0.31/concepts/nodepools /v0.31/concepts/provisioners -/v0.31/concepts/nodeclasses /v0.31/concepts/node-templates -/v0.31/concepts/disruption /v0.31/concepts/deprovisioning -/v0.31/upgrading/upgrade-guide /v0.31/upgrade-guide -/v0.31/reference/instance-types /v0.31/concepts/instance-types -/v0.31/reference/metrics /v0.31/concepts/metrics -/v0.31/reference/settings /v0.31/concepts/settings -/v0.31/reference/threat-model /v0.31/concepts/threat-model - # Redirect all v0.32 documentation to the new routes /v0.32/concepts/provisioners /v0.32/concepts/nodepools /v0.32/concepts/node-templates /v0.32/concepts/nodeclasses diff --git a/website/static/nodeclaims.png b/website/static/nodeclaims.png new file mode 100644 index 000000000000..4a9d84d6ef09 Binary files /dev/null and b/website/static/nodeclaims.png differ