From c556e000d827e2191513bf468274a02bbed055fe Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 19 Aug 2024 15:06:04 -0400 Subject: [PATCH 01/79] support running e2e tests on aws --- Makefile | 4 +- go.mod | 53 +-- go.sum | 57 +++ test/e2e/config/ck8s-aws.yaml | 117 ++++++ test/e2e/create_test.go | 2 +- .../infrastructure-aws/cluster-template.yaml | 351 ++++++++++++++++++ .../e2e/data/shared/v1beta1/metadata-aws.yaml | 9 + test/e2e/e2e_suite_test.go | 2 + 8 files changed, 568 insertions(+), 27 deletions(-) create mode 100644 test/e2e/config/ck8s-aws.yaml create mode 100644 test/e2e/data/infrastructure-aws/cluster-template.yaml create mode 100644 test/e2e/data/shared/v1beta1/metadata-aws.yaml diff --git a/Makefile b/Makefile index 74a1612b..ecf8229f 100644 --- a/Makefile +++ b/Makefile @@ -92,14 +92,14 @@ KUSTOMIZE := $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER) # Ginkgo TEST_DIR := $(shell pwd)/test ARTIFACTS ?= $(shell pwd)/_artifacts -GINKGO_FOCUS ?= +GINKGO_FOCUS ?= Workload cluster creation GINKGO_SKIP ?= GINKGO_NODES ?= 1 # GINKGO_NODES is the number of parallel nodes to run # when running the e2e tests, 1 means no parallelism GINKGO_TIMEOUT ?= 2h GINKGO_POLL_PROGRESS_AFTER ?= 60m GINKGO_POLL_PROGRESS_INTERVAL ?= 5m -E2E_CONF_FILE ?= $(TEST_DIR)/e2e/config/ck8s-docker.yaml +E2E_CONF_FILE ?= $(TEST_DIR)/e2e/config/ck8s-aws.yaml SKIP_RESOURCE_CLEANUP ?= false USE_EXISTING_CLUSTER ?= false GINKGO_NOCOLOR ?= false diff --git a/go.mod b/go.mod index 4e11c408..19c59ca1 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22.6 require ( github.com/canonical/k8s-snap-api v1.0.11 github.com/go-logr/logr v1.4.1 - github.com/google/uuid v1.4.0 + github.com/google/uuid v1.6.0 github.com/onsi/ginkgo v1.16.5 github.com/onsi/ginkgo/v2 v2.17.1 github.com/onsi/gomega v1.32.0 @@ -15,7 +15,7 @@ require ( k8s.io/apimachinery v0.29.3 k8s.io/apiserver v0.29.3 k8s.io/client-go v0.29.3 - k8s.io/klog/v2 v2.110.1 + k8s.io/klog/v2 v2.120.1 k8s.io/utils v0.0.0-20240102154912-e7106e64919e sigs.k8s.io/cluster-api v1.7.1 sigs.k8s.io/cluster-api/test v1.7.1 @@ -24,26 +24,32 @@ require ( ) require ( - github.com/BurntSushi/toml v1.0.0 // indirect + github.com/aws/aws-sdk-go v1.51.17 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + sigs.k8s.io/cluster-api-provider-aws/v2 v2.6.1 // indirect +) + +require ( + github.com/BurntSushi/toml v1.2.1 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.2.0 // indirect + github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Microsoft/go-winio v0.5.0 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/adrg/xdg v0.4.0 // indirect - github.com/alessio/shellescape v1.4.1 // indirect + github.com/alessio/shellescape v1.4.2 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect - github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect + github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.5.0 // indirect github.com/docker/docker v25.0.6+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect - github.com/docker/go-units v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect @@ -60,7 +66,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/cel-go v0.17.7 // indirect + github.com/google/cel-go v0.17.8 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-github/v53 v53.2.0 // indirect @@ -69,7 +75,7 @@ require ( github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/huandu/xstrings v1.3.3 // indirect + github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -88,14 +94,13 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.2 // indirect + github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.18.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/client_golang v1.19.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.52.2 // indirect + github.com/prometheus/procfs v0.13.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect @@ -103,7 +108,7 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cobra v1.8.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace // indirect github.com/spf13/viper v1.18.2 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect @@ -114,17 +119,17 @@ require ( go.opentelemetry.io/otel/trace v1.22.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/mod v0.19.0 - golang.org/x/net v0.23.0 // indirect - golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/oauth2 v0.19.0 // indirect golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/term v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.17.0 // indirect + golang.org/x/tools v0.18.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect @@ -136,7 +141,7 @@ require ( k8s.io/apiextensions-apiserver v0.29.3 // indirect k8s.io/cluster-bootstrap v0.29.3 // indirect k8s.io/component-base v0.29.3 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kind v0.22.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index 33ea1288..660df96f 100644 --- a/go.sum +++ b/go.sum @@ -2,12 +2,16 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU= @@ -18,12 +22,18 @@ github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= +github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= +github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/aws/aws-sdk-go v1.51.17 h1:Cfa40lCdjv9OxC3X1Ks3a6O1Tu3gOANSyKHOSw/zuWU= +github.com/aws/aws-sdk-go v1.51.17/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -33,8 +43,11 @@ github.com/canonical/k8s-snap-api v1.0.11 h1:nGtwrUQBLiaL3HUXFx2gb4kq6qVpl2yNwMw github.com/canonical/k8s-snap-api v1.0.11/go.mod h1:LDPoIYCeYnfgOFrwVPJ/4edGU264w7BB7g0GsVi36AY= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -67,6 +80,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 h1:7QPwrLT79GlD5sizHf27aoY2RTvw62mO6x7mxkScNk0= github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46/go.mod h1:esf2rsHFNlZlxsqsZDojNBcnNs5REqIvRrWRHqX0vEU= github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= @@ -120,6 +135,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/cel-go v0.17.7 h1:6ebJFzu1xO2n7TLtN+UBqShGBhlD85bhvglh5DpcfqQ= github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= +github.com/google/cel-go v0.17.8 h1:j9m730pMZt1Fc4oKhCLUHfjj6527LuhYcYw0Rl8gqto= +github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -145,6 +162,8 @@ github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2/go.mod h1:Tv1PlzqC github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -156,6 +175,8 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= +github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= @@ -164,6 +185,9 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -227,6 +251,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8= +github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -240,12 +266,20 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/common v0.52.2 h1:LW8Vk7BccEdONfrJBDffQGRtpSzi5CQaRZGtboOO2ck= +github.com/prometheus/common v0.52.2/go.mod h1:lrWtQx+iDfn2mbH5GUzlH9TSHyfZpHkSiG1W7y3sF2Q= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o= +github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -259,6 +293,7 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= @@ -271,6 +306,8 @@ github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace h1:9PNP1jnUjRhfmGMlkXHjYPishpcw4jpSt/V/xYY3FMA= +github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= @@ -288,6 +325,7 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= @@ -334,6 +372,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -352,8 +392,12 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg= +golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -384,11 +428,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -406,6 +454,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -471,14 +521,21 @@ k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo= k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio= k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 h1:/U5vjBbQn3RChhv7P11uhYvCSm5G2GaIi5AIGBS6r4c= sigs.k8s.io/cluster-api v1.7.1 h1:JkMAbAMzBM+WBHxXLTJXTiCisv1PAaHRzld/3qrmLYY= sigs.k8s.io/cluster-api v1.7.1/go.mod h1:V9ZhKLvQtsDODwjXOKgbitjyCmC71yMBwDcMyNNIov0= +sigs.k8s.io/cluster-api-provider-aws/v2 v2.6.1 h1:vbZUYEB7OfPlfHk6wis+UrvRLTqv5F4Nrjl2WDJ1kiw= +sigs.k8s.io/cluster-api-provider-aws/v2 v2.6.1/go.mod h1:1aq1EZbirRW6NC2gYUFCc7cVFwX9PM/vDvoU+2oGPuw= sigs.k8s.io/cluster-api/test v1.7.1 h1:QDru2586ZjIFBTW1Z7VVXVtauzR/yANm4tglUNLm9iE= sigs.k8s.io/cluster-api/test v1.7.1/go.mod h1:yG0g5Mdq73fMn9JP4akgRQPSne973L+Qx6iVH+LjtSM= sigs.k8s.io/controller-runtime v0.17.3 h1:65QmN7r3FWgTxDMz9fvGnO1kbf2nu+acg9p2R9oYYYk= diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml new file mode 100644 index 00000000..cd27fbdc --- /dev/null +++ b/test/e2e/config/ck8s-aws.yaml @@ -0,0 +1,117 @@ +--- +# E2E test scenario using local dev images and manifests built from the source tree for following providers: +# - cluster-api +# - bootstrap ck8s +# - control-plane ck8s +# - aws +images: + # Use local dev images built source tree; + - name: ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev + loadBehavior: mustLoad + - name: ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev + loadBehavior: mustLoad + +providers: + - name: cluster-api + type: CoreProvider + versions: + - name: v1.6.2 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.2/core-components.yaml + type: url + files: + - sourcePath: "../data/shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - name: aws + type: InfrastructureProvider + versions: + # By default, will use the latest version defined in ../data/shared/v1beta1/metadata.yaml + # to init the management cluster + - name: v2.6.1 # used during e2e-test + value: https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.6.1/infrastructure-components.yaml + type: url + files: + - sourcePath: "../data/shared/v1beta1/metadata-aws.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + + # Add v1.7.99 to support tilt (not presented in ../data/shared/v1beta1/metadata.yaml) + # when bootstrapping with tilt, it will use + # the defaultProviderVersion in https://github.com/kubernetes-sigs/cluster-api/blob/main/hack/tools/internal/tilt-prepare/main.go as + # default version for docker infrastructure provider + # name here should match defaultProviderVersion + - name: v1.7.99 # next; use manifest from source files + value: https://github.com/kubernetes-sigs/cluster-api/releases/latest/download/infrastructure-components-development.yaml + type: url + files: + - sourcePath: "../data/shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + files: + - sourcePath: "../data/infrastructure-aws/cluster-template.yaml" + - name: ck8s + type: BootstrapProvider + versions: + # Could add older release version for upgrading test, but + # by default, will only use the latest version defined in + # ${ProjectRoot}/metadata.yaml to init the management cluster + # this version should be updated when ${ProjectRoot}/metadata.yaml + # is modified + - name: v0.1.99 # next; use manifest from source files + value: "../../../bootstrap/config/default" + replacements: + - old: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:latest" + new: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev" + files: + - sourcePath: "../../../metadata.yaml" + targetName: "metadata.yaml" + - name: ck8s + type: ControlPlaneProvider + versions: + - name: v0.1.99 # next; use manifest from source files + value: "../../../controlplane/config/default" + replacements: + - old: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:latest" + new: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev" + files: + - sourcePath: "../../../metadata.yaml" + targetName: "metadata.yaml" + +variables: + KUBERNETES_VERSION_MANAGEMENT: "v1.28.0" + KUBERNETES_VERSION: "v1.30.0" + KUBERNETES_VERSION_UPGRADE_TO: "v1.30.1" + IP_FAMILY: "IPv4" + KIND_IMAGE_VERSION: "v1.28.0" + AWS_CONTROL_PLANE_MACHINE_TYPE: t3.large + AWS_NODE_MACHINE_TYPE: t3.large + AWS_SSH_KEY_NAME: "default" + AWS_CCM_IMAGE: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3" + # https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml#L203C1-L205C27 + EXP_CLUSTER_RESOURCE_SET: "true" + EXP_MACHINE_POOL: "true" + CLUSTER_TOPOLOGY: "true" + +intervals: + # copied from https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml + default/wait-cluster: ["35m", "10s"] + default/wait-control-plane: ["35m", "10s"] + default/wait-worker-nodes: ["20m", "10s"] + conformance/wait-control-plane: ["35m", "10s"] + conformance/wait-worker-nodes: ["35m", "10s"] + default/wait-controllers: ["5m", "10s"] + default/wait-delete-cluster: ["20m", "10s"] + default/wait-machine-upgrade: ["35m", "10s"] + default/wait-contolplane-upgrade: ["40m", "10s"] + default/wait-machine-status: ["25m", "10s"] + default/wait-failed-machine-status: ["2m", "10s"] + default/wait-infra-subnets: ["5m", "30s"] + default/wait-machine-pool-nodes: ["40m", "10s"] + default/wait-machine-pool-upgrade: ["50m", "10s"] + default/wait-create-identity: ["1m", "10s"] + default/wait-job: ["10m", "10s"] + default/wait-deployment-ready: ["5m", "10s"] + default/wait-loadbalancer-ready: ["5m", "30s"] diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go index 81dd2b8f..63497e17 100644 --- a/test/e2e/create_test.go +++ b/test/e2e/create_test.go @@ -48,7 +48,7 @@ var _ = Describe("Workload cluster creation", func() { Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion)) clusterName = fmt.Sprintf("capick8s-create-%s", util.RandomString(6)) - infrastructureProvider = "docker" + infrastructureProvider = "aws" // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) diff --git a/test/e2e/data/infrastructure-aws/cluster-template.yaml b/test/e2e/data/infrastructure-aws/cluster-template.yaml new file mode 100644 index 00000000..1ae700bb --- /dev/null +++ b/test/e2e/data/infrastructure-aws/cluster-template.yaml @@ -0,0 +1,351 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.1.0.0/16 + services: + cidrBlocks: + - 10.152.183.0/24 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSCluster +metadata: + name: ${CLUSTER_NAME} +spec: + region: ${AWS_REGION} + sshKeyName: ${AWS_SSH_KEY_NAME} + bastion: + enabled: ${AWS_CREATE_BASTION:=true} + controlPlaneLoadBalancer: + healthCheckProtocol: TCP + network: + cni: + cniIngressRules: + - description: microcluster + protocol: tcp + toPort: 2380 +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: ${CLUSTER_NAME}-control-plane + spec: + files: + # note(ben): This is only required as long as k8s does not have a stable release. + - path: /capi/scripts/install.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + snap install k8s --classic --edge + - path: /capi/scripts/bootstrap.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + if [ ! -f /etc/kubernetes/pki/ca.crt ]; then + k8s bootstrap --name '{{ ds.meta_data.local_hostname }}' --address "$(cat /capi/etc/microcluster-address)" --file '/capi/etc/config.yaml' + fi + - path: /capi/scripts/join-cluster.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + k8s join-cluster --name '{{ ds.meta_data.local_hostname }}' "$(cat /capi/etc/join-token)" --address "$(cat /capi/etc/microcluster-address)" --file "/capi/etc/config.yaml" + controlPlane: + cloudProvider: external + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + ami: + id: ${AWS_AMI_ID:=ami-0ad50e72a79228704} + iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_CONTROL_PLANE_INSTANCE_TYPE:=t3.large} + publicIP: ${AWS_PUBLIC_IP:=false} + sshKeyName: ${AWS_SSH_KEY_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-worker-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + template: + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: CK8sConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + ami: + id: ${AWS_AMI_ID:=ami-0ad50e72a79228704} + iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_NODE_INSTANCE_TYPE:=t3.large} + publicIP: ${AWS_PUBLIC_IP:=false} + sshKeyName: ${AWS_SSH_KEY_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + files: + # note(ben): This is only required as long as k8s does not have a stable release. + - path: /capi/scripts/install.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + snap install k8s --classic --edge + - path: /capi/scripts/bootstrap.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + if [ ! -f /etc/kubernetes/pki/ca.crt ]; then + k8s bootstrap --name '{{ ds.meta_data.local_hostname }}' --address "$(cat /capi/etc/microcluster-address)" --file '/capi/etc/config.yaml' + fi + - path: /capi/scripts/join-cluster.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + k8s join-cluster --name '{{ ds.meta_data.local_hostname }}' "$(cat /capi/etc/join-token)" --address "$(cat /capi/etc/microcluster-address)" --file "/capi/etc/config.yaml" + +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: ${AWS_CCM_IMAGE} + args: + - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + - --configure-cloud-routes=false + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + name: cloud-controller-manager-addon + namespace: default diff --git a/test/e2e/data/shared/v1beta1/metadata-aws.yaml b/test/e2e/data/shared/v1beta1/metadata-aws.yaml new file mode 100644 index 00000000..8b0680e2 --- /dev/null +++ b/test/e2e/data/shared/v1beta1/metadata-aws.yaml @@ -0,0 +1,9 @@ +# maps release series of major.minor to cluster-api contract version, +# update this file only when you update the version for cluster-api +# CoreProvider and docker InfrastructureProvider in test/e2e/config/k3s-docker.yaml +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +kind: Metadata +releaseSeries: + - major: 2 + minor: 6 + contract: v1beta1 diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 9dd78e93..509d7435 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -39,6 +39,7 @@ import ( "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" dockerinfrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1" + awsinfrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" ctrl "sigs.k8s.io/controller-runtime" bootstrapv1 "github.com/canonical/cluster-api-k8s/bootstrap/api/v1beta2" @@ -196,6 +197,7 @@ func initScheme() *runtime.Scheme { Expect(controlplanev1.AddToScheme(sc)).To(Succeed()) Expect(bootstrapv1.AddToScheme(sc)).To(Succeed()) Expect(dockerinfrav1.AddToScheme(sc)).To(Succeed()) + Expect(awsinfrav1.AddToScheme(sc)).To(Succeed()) return sc } From e0c2d45fde57cb8df4d7ba46221642eafd7d3b88 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 19 Aug 2024 15:43:51 -0400 Subject: [PATCH 02/79] add ami id --- test/e2e/config/ck8s-aws.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index cd27fbdc..0dfd18ca 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -88,7 +88,8 @@ variables: KIND_IMAGE_VERSION: "v1.28.0" AWS_CONTROL_PLANE_MACHINE_TYPE: t3.large AWS_NODE_MACHINE_TYPE: t3.large - AWS_SSH_KEY_NAME: "default" + AWS_SSH_KEY_NAME: "etienne" + AWS_AMI_ID: "ami-05145146e3a9db6f3" AWS_CCM_IMAGE: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3" # https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml#L203C1-L205C27 EXP_CLUSTER_RESOURCE_SET: "true" From ee591512c959f634dd5d67ce8a6945ed092adf7f Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 20 Aug 2024 21:06:01 -0400 Subject: [PATCH 03/79] hacks to get the cluster deployed --- .../infrastructure-aws/cluster-template.yaml | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test/e2e/data/infrastructure-aws/cluster-template.yaml b/test/e2e/data/infrastructure-aws/cluster-template.yaml index 1ae700bb..a9754743 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template.yaml @@ -2,6 +2,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: name: ${CLUSTER_NAME} + labels: + ccm: external spec: clusterNetwork: pods: @@ -41,7 +43,6 @@ apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane metadata: name: ${CLUSTER_NAME}-control-plane - namespace: default spec: machineTemplate: infrastructureTemplate: @@ -49,6 +50,11 @@ spec: kind: AWSMachineTemplate name: ${CLUSTER_NAME}-control-plane spec: + preRunCommands: + - systemctl stop kubelet || true + - systemctl disable kubelet || true + - systemctl stop containerd || true + - systemctl disable containerd || true files: # note(ben): This is only required as long as k8s does not have a stable release. - path: /capi/scripts/install.sh @@ -84,7 +90,7 @@ spec: template: spec: ami: - id: ${AWS_AMI_ID:=ami-0ad50e72a79228704} + id: ${AWS_AMI_ID} iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io instanceType: ${AWS_CONTROL_PLANE_INSTANCE_TYPE:=t3.large} publicIP: ${AWS_PUBLIC_IP:=false} @@ -122,7 +128,7 @@ spec: template: spec: ami: - id: ${AWS_AMI_ID:=ami-0ad50e72a79228704} + id: ${AWS_AMI_ID} iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io instanceType: ${AWS_NODE_INSTANCE_TYPE:=t3.large} publicIP: ${AWS_PUBLIC_IP:=false} @@ -135,6 +141,11 @@ metadata: spec: template: spec: + preRunCommands: + - systemctl stop kubelet || true + - systemctl disable kubelet || true + - systemctl stop containerd || true + - systemctl disable containerd || true files: # note(ben): This is only required as long as k8s does not have a stable release. - path: /capi/scripts/install.sh @@ -166,7 +177,7 @@ metadata: spec: clusterSelector: matchLabels: - cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + ccm: external resources: - kind: ConfigMap name: cloud-controller-manager-addon @@ -348,4 +359,3 @@ data: kind: ConfigMap metadata: name: cloud-controller-manager-addon - namespace: default From ddb9a101ce283688932bf34341d17b27329642b7 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Wed, 21 Aug 2024 14:12:09 -0400 Subject: [PATCH 04/79] newest template updates --- test/e2e/config/ck8s-aws.yaml | 2 + .../infrastructure-aws/cluster-template.yaml | 57 +++++-------------- 2 files changed, 15 insertions(+), 44 deletions(-) diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index 0dfd18ca..6f5f0a19 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -90,6 +90,8 @@ variables: AWS_NODE_MACHINE_TYPE: t3.large AWS_SSH_KEY_NAME: "etienne" AWS_AMI_ID: "ami-05145146e3a9db6f3" + AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE: 16 + AWS_NODE_ROOT_VOLUME_SIZE: 16 AWS_CCM_IMAGE: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3" # https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml#L203C1-L205C27 EXP_CLUSTER_RESOURCE_SET: "true" diff --git a/test/e2e/data/infrastructure-aws/cluster-template.yaml b/test/e2e/data/infrastructure-aws/cluster-template.yaml index a9754743..66f83b7c 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template.yaml @@ -29,7 +29,7 @@ spec: region: ${AWS_REGION} sshKeyName: ${AWS_SSH_KEY_NAME} bastion: - enabled: ${AWS_CREATE_BASTION:=true} + enabled: ${AWS_CREATE_BASTION} controlPlaneLoadBalancer: healthCheckProtocol: TCP network: @@ -43,6 +43,7 @@ apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane metadata: name: ${CLUSTER_NAME}-control-plane + namespace: default spec: machineTemplate: infrastructureTemplate: @@ -50,11 +51,7 @@ spec: kind: AWSMachineTemplate name: ${CLUSTER_NAME}-control-plane spec: - preRunCommands: - - systemctl stop kubelet || true - - systemctl disable kubelet || true - - systemctl stop containerd || true - - systemctl disable containerd || true + nodeName: "{{ ds.meta_data.local_hostname }}" files: # note(ben): This is only required as long as k8s does not have a stable release. - path: /capi/scripts/install.sh @@ -63,20 +60,6 @@ spec: content: | #!/bin/bash -xe snap install k8s --classic --edge - - path: /capi/scripts/bootstrap.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - if [ ! -f /etc/kubernetes/pki/ca.crt ]; then - k8s bootstrap --name '{{ ds.meta_data.local_hostname }}' --address "$(cat /capi/etc/microcluster-address)" --file '/capi/etc/config.yaml' - fi - - path: /capi/scripts/join-cluster.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - k8s join-cluster --name '{{ ds.meta_data.local_hostname }}' "$(cat /capi/etc/join-token)" --address "$(cat /capi/etc/microcluster-address)" --file "/capi/etc/config.yaml" controlPlane: cloudProvider: external replicas: ${CONTROL_PLANE_MACHINE_COUNT} @@ -92,9 +75,11 @@ spec: ami: id: ${AWS_AMI_ID} iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io - instanceType: ${AWS_CONTROL_PLANE_INSTANCE_TYPE:=t3.large} - publicIP: ${AWS_PUBLIC_IP:=false} + instanceType: ${AWS_CONTROL_PLANE_INSTANCE_TYPE} + publicIP: ${AWS_PUBLIC_IP} sshKeyName: ${AWS_SSH_KEY_NAME} + rootVolume: + size: ${AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -130,9 +115,11 @@ spec: ami: id: ${AWS_AMI_ID} iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io - instanceType: ${AWS_NODE_INSTANCE_TYPE:=t3.large} - publicIP: ${AWS_PUBLIC_IP:=false} + instanceType: ${AWS_NODE_INSTANCE_TYPE} + publicIP: ${AWS_PUBLIC_IP} sshKeyName: ${AWS_SSH_KEY_NAME} + rootVolume: + size: ${AWS_NODE_ROOT_VOLUME_SIZE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate @@ -141,11 +128,7 @@ metadata: spec: template: spec: - preRunCommands: - - systemctl stop kubelet || true - - systemctl disable kubelet || true - - systemctl stop containerd || true - - systemctl disable containerd || true + nodeName: "{{ ds.meta_data.local_hostname }}" files: # note(ben): This is only required as long as k8s does not have a stable release. - path: /capi/scripts/install.sh @@ -154,21 +137,6 @@ spec: content: | #!/bin/bash -xe snap install k8s --classic --edge - - path: /capi/scripts/bootstrap.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - if [ ! -f /etc/kubernetes/pki/ca.crt ]; then - k8s bootstrap --name '{{ ds.meta_data.local_hostname }}' --address "$(cat /capi/etc/microcluster-address)" --file '/capi/etc/config.yaml' - fi - - path: /capi/scripts/join-cluster.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - k8s join-cluster --name '{{ ds.meta_data.local_hostname }}' "$(cat /capi/etc/join-token)" --address "$(cat /capi/etc/microcluster-address)" --file "/capi/etc/config.yaml" - --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet @@ -359,3 +327,4 @@ data: kind: ConfigMap metadata: name: cloud-controller-manager-addon + namespace: default From 1c67a636fbfb4c27dcf4c2b1b2002256bbc01e8d Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Mon, 8 Jul 2024 13:04:35 +0200 Subject: [PATCH 05/79] Add clusterctl config --- docs/clusterctl.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/clusterctl.yaml diff --git a/docs/clusterctl.yaml b/docs/clusterctl.yaml new file mode 100644 index 00000000..0c2e052c --- /dev/null +++ b/docs/clusterctl.yaml @@ -0,0 +1,8 @@ +--- +providers: + - name: ck8s + type: BootstrapProvider + url: "https://github.com/canonical/cluster-api-k8s/releases/latest/bootstrap-components.yaml" + - name: ck8s + type: ControlPlaneProvider + url: "https://github.com/canonical/cluster-api-k8s/releases/latest/control-plane-components.yaml" From 846c7eae42ce459cf11049e41461badf97bec5c4 Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Wed, 10 Jul 2024 10:20:40 +0200 Subject: [PATCH 06/79] docs --- docs/aws.md | 3 + docs/overview.md | 149 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 docs/aws.md create mode 100644 docs/overview.md diff --git a/docs/aws.md b/docs/aws.md new file mode 100644 index 00000000..1d59f820 --- /dev/null +++ b/docs/aws.md @@ -0,0 +1,3 @@ +# Setting up the AWS Infrastructure provider + +This How-To walks you to the steps to set up CAPI with the AWS infrastructure provider diff --git a/docs/overview.md b/docs/overview.md new file mode 100644 index 00000000..368b099e --- /dev/null +++ b/docs/overview.md @@ -0,0 +1,149 @@ +# Cluster Provisioning with CAPI + +This guide covers how to deploy a Canonical Kubernetes multi-node cluster using Cluster API (CAPI). + +## Install `clusterctl` + +The `clusterctl` CLI tool manages the lifecycle of a Cluster API management cluster. To install it, follow the [upstream instructions]. Typically, this involves fetching the executable that matches your hardware architecture and placing it in your PATH. For example, at the time this guide was written, for `amd64` you would: + +```sh +curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.3/clusterctl-linux-amd64 -o clusterctl +sudo install -o root -g root -m 0755 clusterctl /usr/local/bin/clusterctl +``` + +### Set up a management Cluster + +The management cluster hosts the CAPI providers. You can use a Canonical Kubernetes cluster as a management cluster: + +```sh +sudo snap install k8s --classic +sudo k8s bootstrap +sudo k8s status --wait-ready +mkdir -p ~/.kube/ +sudo k8s config > ~/.kube/config +``` + +When setting up the management cluster, place its kubeconfig under `~/.kube/config` so other tools such as `clusterctl` can discover and interact with it. + +### Prepare the Infrastructure Provider + +Before generating a cluster, you need to configure the infrastructure provider. Each provider has its own prerequisites. Please follow the Cluster API instructions for the additional infrastructure-specific configuration. + +#### Example Using AWS + +The AWS infrastructure provider requires the `clusterawsadm` tool to be installed: + +```sh +curl -L https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.0.2/clusterawsadm-linux-amd64 -o clusterawsadm +chmod +x clusterawsadm +sudo mv clusterawsadm /usr/local/bin +``` + +With `clusterawsadm`, you can bootstrap the AWS environment that CAPI will use. + +Start by setting up environment variables defining the AWS account to use, if these are not already defined: + +```sh +export AWS_REGION= +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +``` + +If you are using multi-factor authentication, you will also need: + +```sh +export AWS_SESSION_TOKEN= # If you are using Multi-Factor Auth. +``` + +The `clusterawsadm` uses these details to create a CloudFormation stack in your AWS account with the correct IAM resources: + +```sh +clusterawsadm bootstrap iam create-cloudformation-stack +``` + +The credentials should also be encoded and stored as a Kubernetes secret: + +```sh +export AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile) +``` + +### Initialize the Management Cluster + +To initialize the management cluster with the latest released version of the providers and the infrastructure of your choice: + +```sh +clusterctl init --bootstrap ck8s --control-plane ck8s -i +``` + +### Generate a Cluster Spec Manifest + +Once the bootstrap and control-plane controllers are up and running, you can apply the cluster manifests with the specifications of the cluster you want to provision. + +For Canonical Kubernetes, there are example manifests in the bootstrap provider examples directory on GitHub. + +Alternatively, you can generate a cluster manifest for a selected set of commonly used infrastructures via templates provided by the Canonical Kubernetes team. Visit the usage instructions for a list of different providers and their deployment. + +Ensure you have initialized the desired infrastructure provider and fetch the Canonical Kubernetes bootstrap provider repository: + +```sh +git clone https://github.com/canonical/cluster-api-bootstrap-provider-microk8s +``` + +Review the list of variables needed for the cluster template: + +```sh +cd cluster-api-bootstrap-provider-microk8s +clusterctl generate cluster k8s- --from ./templates/cluster-template-.yaml --list-variables +``` + +Set the respective environment variables by editing the rc file as needed before sourcing it. Then generate the cluster manifest: + +```sh +source ./templates/cluster-template-.rc +clusterctl generate cluster k8s- --from ./templates/cluster-template-.yaml > cluster.yaml +``` + +Each provisioned node is associated with a `K8sConfig`, through which you can set the cluster’s properties. Review the available options in the respective definitions file and edit the cluster manifest (`cluster.yaml` above) to match your needs. Note that the configuration structure is similar to that of `kubeadm` - in the `K8sConfig`, you will find a `ClusterConfiguration` and an `InitConfiguration` section. + +### Deploy the Cluster + +To deploy the cluster, run: + +```sh +sudo microk8s kubectl apply -f cluster.yaml +``` + +To see the deployed machines: + +```sh +sudo microk8s kubectl get machine +``` + +After the first control plane node is provisioned, you can get the kubeconfig of the workload cluster: + +```sh +clusterctl get kubeconfig > kubeconfig +``` + +You can then see the workload nodes using: + +```sh +KUBECONFIG=./kubeconfig kubectl get node +``` + +### Delete the Cluster + +To get the list of provisioned clusters: + +```sh +sudo microk8s kubectl get clusters +``` + +To delete a cluster: + +```sh +sudo microk8s kubectl delete cluster +``` + + +[upstream instructions]: https://cluster-api.sigs.k8s.io/user/quick-start#install-clusterctl From 7a695a4051a63ffe37459c3d3d44529bda22eecc Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Thu, 11 Jul 2024 14:00:12 +0200 Subject: [PATCH 07/79] move clusterctl and update template-variables --- docs/clusterctl.yaml => clusterctl.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/clusterctl.yaml => clusterctl.yaml (100%) diff --git a/docs/clusterctl.yaml b/clusterctl.yaml similarity index 100% rename from docs/clusterctl.yaml rename to clusterctl.yaml From f48bfd779dc8511f9fd1297c760056dbe9676c1a Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Fri, 12 Jul 2024 06:18:22 +0200 Subject: [PATCH 08/79] delete obsolete docs --- docs/aws.md | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 docs/aws.md diff --git a/docs/aws.md b/docs/aws.md deleted file mode 100644 index 1d59f820..00000000 --- a/docs/aws.md +++ /dev/null @@ -1,3 +0,0 @@ -# Setting up the AWS Infrastructure provider - -This How-To walks you to the steps to set up CAPI with the AWS infrastructure provider From 0931a0f39e3abe01ca60fa528cde1639f652d2d1 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 20 Aug 2024 21:42:41 -0400 Subject: [PATCH 09/79] remove clusterctl --- clusterctl.yaml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 clusterctl.yaml diff --git a/clusterctl.yaml b/clusterctl.yaml deleted file mode 100644 index 0c2e052c..00000000 --- a/clusterctl.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -providers: - - name: ck8s - type: BootstrapProvider - url: "https://github.com/canonical/cluster-api-k8s/releases/latest/bootstrap-components.yaml" - - name: ck8s - type: ControlPlaneProvider - url: "https://github.com/canonical/cluster-api-k8s/releases/latest/control-plane-components.yaml" From a0ad6a56c4d8f18237a2b911215cf73012e5e6ae Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 20 Aug 2024 21:54:06 -0400 Subject: [PATCH 10/79] review comments no hardcoded defaults, empty variables by default, use new nodeName field to specify node name --- templates/aws/template-variables.rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/aws/template-variables.rc b/templates/aws/template-variables.rc index 5939f9da..ed7b8cc7 100644 --- a/templates/aws/template-variables.rc +++ b/templates/aws/template-variables.rc @@ -4,7 +4,7 @@ export CONTROL_PLANE_MACHINE_COUNT=3 export WORKER_MACHINE_COUNT=3 # AWS region -export AWS_REGION="eu-central-1" +export AWS_REGION="" # e.g. us-east-2 # AWS machine configuration export AWS_CREATE_BASTION="true" From 72187a93a0c7f07792932b54ffd410d9628fe054 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 20 Aug 2024 21:55:25 -0400 Subject: [PATCH 11/79] reformat --- templates/aws/template-variables.rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/aws/template-variables.rc b/templates/aws/template-variables.rc index ed7b8cc7..f636ff72 100644 --- a/templates/aws/template-variables.rc +++ b/templates/aws/template-variables.rc @@ -4,7 +4,7 @@ export CONTROL_PLANE_MACHINE_COUNT=3 export WORKER_MACHINE_COUNT=3 # AWS region -export AWS_REGION="" # e.g. us-east-2 +export AWS_REGION="" # e.g. "us-east-2" # AWS machine configuration export AWS_CREATE_BASTION="true" From a90793e4176297f66bc5216fbf33770a9a2596dc Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 20 Aug 2024 22:02:09 -0400 Subject: [PATCH 12/79] remove microk8s references --- docs/overview.md | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/docs/overview.md b/docs/overview.md index 368b099e..cdcace65 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -16,11 +16,11 @@ sudo install -o root -g root -m 0755 clusterctl /usr/local/bin/clusterctl The management cluster hosts the CAPI providers. You can use a Canonical Kubernetes cluster as a management cluster: ```sh -sudo snap install k8s --classic +sudo snap install k8s --classic --edge sudo k8s bootstrap sudo k8s status --wait-ready mkdir -p ~/.kube/ -sudo k8s config > ~/.kube/config +sudo k8s kubectl config view --raw > ~/.kube/config ``` When setting up the management cluster, place its kubeconfig under `~/.kube/config` so other tools such as `clusterctl` can discover and interact with it. @@ -44,7 +44,7 @@ With `clusterawsadm`, you can bootstrap the AWS environment that CAPI will use. Start by setting up environment variables defining the AWS account to use, if these are not already defined: ```sh -export AWS_REGION= +export AWS_REGION= export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= ``` @@ -79,20 +79,14 @@ clusterctl init --bootstrap ck8s --control-plane ck8s -i clusterctl generate cluster k8s- --from ./templates/cluster-template-.yaml --list-variables ``` @@ -103,20 +97,20 @@ source ./templates/cluster-template-.rc clusterctl generate cluster k8s- --from ./templates/cluster-template-.yaml > cluster.yaml ``` -Each provisioned node is associated with a `K8sConfig`, through which you can set the cluster’s properties. Review the available options in the respective definitions file and edit the cluster manifest (`cluster.yaml` above) to match your needs. Note that the configuration structure is similar to that of `kubeadm` - in the `K8sConfig`, you will find a `ClusterConfiguration` and an `InitConfiguration` section. +Each provisioned node is associated with a `K8sConfig`, through which you can set the cluster’s properties. Review the available options in the respective definitions file and edit the cluster manifest (`cluster.yaml` above) to match your needs. Note that the configuration structure is similar to that of `kubeadm` - in the `CK8sConfig`, you will find a `ClusterConfiguration` and an `InitConfiguration` section. ### Deploy the Cluster To deploy the cluster, run: ```sh -sudo microk8s kubectl apply -f cluster.yaml +sudo k8s kubectl apply -f cluster.yaml ``` To see the deployed machines: ```sh -sudo microk8s kubectl get machine +sudo k8s kubectl get machine ``` After the first control plane node is provisioned, you can get the kubeconfig of the workload cluster: @@ -136,13 +130,13 @@ KUBECONFIG=./kubeconfig kubectl get node To get the list of provisioned clusters: ```sh -sudo microk8s kubectl get clusters +sudo k8s kubectl get clusters ``` To delete a cluster: ```sh -sudo microk8s kubectl delete cluster +sudo k8s kubectl delete cluster ``` From a3d9b8569479097ad253015fab13e3fb27ead4d3 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 20 Aug 2024 22:11:13 -0400 Subject: [PATCH 13/79] add ami id to .rc file --- templates/aws/template-variables.rc | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/templates/aws/template-variables.rc b/templates/aws/template-variables.rc index f636ff72..34bc801b 100644 --- a/templates/aws/template-variables.rc +++ b/templates/aws/template-variables.rc @@ -1,20 +1,17 @@ # Kubernetes cluster configuration -export KUBERNETES_VERSION=v1.31.0 -export CONTROL_PLANE_MACHINE_COUNT=3 -export WORKER_MACHINE_COUNT=3 +export KUBERNETES_VERSION=v1.30.0 +export CONTROL_PLANE_MACHINE_COUNT= # e.g. 1 +export WORKER_MACHINE_COUNT= # AWS region export AWS_REGION="" # e.g. "us-east-2" # AWS machine configuration -export AWS_CREATE_BASTION="true" -export AWS_PUBLIC_IP="true" -export AWS_CONTROL_PLANE_INSTANCE_TYPE="t3.large" -export AWS_NODE_INSTANCE_TYPE="t3.large" -export AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE=16 -export AWS_NODE_ROOT_VOLUME_SIZE=16 -export AWS_SSH_KEY_NAME="default" -# List upstream AMIs with clusterawsadm ami list --owner-id 819546954734 -export AWS_AMI_ID="ami-027b534ab5d0b4886" +export AWS_CREATE_BASTION= # e.g. "true" +export AWS_PUBLIC_IP= # e.g. "true" +export AWS_CONTROL_PLANE_MACHINE_FLAVOR= # e.g. "t3.large" +export AWS_NODE_MACHINE_FLAVOR= # e.g. "t3.large" +export AWS_SSH_KEY_NAME= # e.g. "default" +export AWS_AMI_ID= # e.g. "ami-0ad50e72a79228704" export AWS_CCM_IMAGE=registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 From cd4073e64a3826d3d94ea267b8e55222f14e58d5 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Wed, 21 Aug 2024 14:02:47 -0400 Subject: [PATCH 14/79] configurable root volume size. fix var. name --- templates/aws/template-variables.rc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/templates/aws/template-variables.rc b/templates/aws/template-variables.rc index 34bc801b..48313d8a 100644 --- a/templates/aws/template-variables.rc +++ b/templates/aws/template-variables.rc @@ -9,8 +9,10 @@ export AWS_REGION="" # e.g. "us-east-2" # AWS machine configuration export AWS_CREATE_BASTION= # e.g. "true" export AWS_PUBLIC_IP= # e.g. "true" -export AWS_CONTROL_PLANE_MACHINE_FLAVOR= # e.g. "t3.large" -export AWS_NODE_MACHINE_FLAVOR= # e.g. "t3.large" +export AWS_CONTROL_PLANE_INSTANCE_TYPE= # e.g. "t3.large" +export AWS_NODE_INSTANCE_TYPE= # e.g. "t3.large" +export AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE= # in Gi, e.g. 16 +export AWS_NODE_ROOT_VOLUME_SIZE= # in Gi, e.g. 16 export AWS_SSH_KEY_NAME= # e.g. "default" export AWS_AMI_ID= # e.g. "ami-0ad50e72a79228704" From 1d08d54b9577699ddc08112c91d6e322ae4ef666 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Fri, 23 Aug 2024 21:04:22 -0400 Subject: [PATCH 15/79] proper e2e config support tilt with tilt-provider.json --- test/e2e/config/ck8s-aws.yaml | 79 +++++++++++-------- .../e2e/data/shared/v1beta1/metadata-aws.yaml | 9 --- test/e2e/data/shared/v1beta1/metadata.yaml | 8 +- .../e2e/data/shared/v1beta1_aws/metadata.yaml | 5 ++ tilt-provider.yaml | 24 ++++++ 5 files changed, 77 insertions(+), 48 deletions(-) delete mode 100644 test/e2e/data/shared/v1beta1/metadata-aws.yaml create mode 100644 test/e2e/data/shared/v1beta1_aws/metadata.yaml create mode 100644 tilt-provider.yaml diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index 6f5f0a19..eed9e2eb 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -1,4 +1,6 @@ --- +managementClusterName: capi-test + # E2E test scenario using local dev images and manifests built from the source tree for following providers: # - cluster-api # - bootstrap ck8s @@ -15,9 +17,10 @@ providers: - name: cluster-api type: CoreProvider versions: - - name: v1.6.2 - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.2/core-components.yaml + - name: v1.7.1 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.1/core-components.yaml type: url + contract: v1beta1 files: - sourcePath: "../data/shared/v1beta1/metadata.yaml" replacements: @@ -29,24 +32,24 @@ providers: # By default, will use the latest version defined in ../data/shared/v1beta1/metadata.yaml # to init the management cluster - name: v2.6.1 # used during e2e-test - value: https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.6.1/infrastructure-components.yaml - type: url + value: "../../../../cluster-api-provider-aws/config/default" + contract: v1beta2 files: - - sourcePath: "../data/shared/v1beta1/metadata-aws.yaml" + - sourcePath: "../data/shared/v1beta1_aws/metadata.yaml" replacements: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" - # Add v1.7.99 to support tilt (not presented in ../data/shared/v1beta1/metadata.yaml) + # Add v1.9.99 to support tilt (not presented in ../data/shared/v1beta1/metadata.yaml) # when bootstrapping with tilt, it will use # the defaultProviderVersion in https://github.com/kubernetes-sigs/cluster-api/blob/main/hack/tools/internal/tilt-prepare/main.go as # default version for docker infrastructure provider # name here should match defaultProviderVersion - - name: v1.7.99 # next; use manifest from source files - value: https://github.com/kubernetes-sigs/cluster-api/releases/latest/download/infrastructure-components-development.yaml - type: url + - name: v1.9.99 + value: "../../../../cluster-api-provider-aws/config/default" + contract: v1beta2 files: - - sourcePath: "../data/shared/v1beta1/metadata.yaml" + - sourcePath: "../data/shared/v1beta1_aws/metadata.yaml" replacements: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" @@ -81,40 +84,46 @@ providers: targetName: "metadata.yaml" variables: - KUBERNETES_VERSION_MANAGEMENT: "v1.28.0" + KUBERNETES_VERSION_MANAGEMENT: "v1.30.0" KUBERNETES_VERSION: "v1.30.0" KUBERNETES_VERSION_UPGRADE_TO: "v1.30.1" IP_FAMILY: "IPv4" - KIND_IMAGE_VERSION: "v1.28.0" - AWS_CONTROL_PLANE_MACHINE_TYPE: t3.large - AWS_NODE_MACHINE_TYPE: t3.large + KIND_IMAGE_VERSION: "v1.30.0" + AWS_CONTROL_PLANE_INSTANCE_TYPE: t3.large + AWS_NODE_INSTANCE_TYPE: t3.large + AWS_PUBLIC_IP: false + AWS_CREATE_BASTION: true AWS_SSH_KEY_NAME: "etienne" AWS_AMI_ID: "ami-05145146e3a9db6f3" AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE: 16 AWS_NODE_ROOT_VOLUME_SIZE: 16 - AWS_CCM_IMAGE: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3" + AWS_CCM_IMAGE: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.30.0" # https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml#L203C1-L205C27 EXP_CLUSTER_RESOURCE_SET: "true" - EXP_MACHINE_POOL: "true" - CLUSTER_TOPOLOGY: "true" + #EXP_RUNTIME_SDK: "true" + EXP_MACHINE_SET_PREFLIGHT_CHECKS: "false" + #EXP_MACHINE_POOL: "true" + #CLUSTER_TOPOLOGY: "true" + CAPA_LOGLEVEL: "4" + intervals: # copied from https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml - default/wait-cluster: ["35m", "10s"] - default/wait-control-plane: ["35m", "10s"] - default/wait-worker-nodes: ["20m", "10s"] - conformance/wait-control-plane: ["35m", "10s"] - conformance/wait-worker-nodes: ["35m", "10s"] - default/wait-controllers: ["5m", "10s"] - default/wait-delete-cluster: ["20m", "10s"] - default/wait-machine-upgrade: ["35m", "10s"] - default/wait-contolplane-upgrade: ["40m", "10s"] - default/wait-machine-status: ["25m", "10s"] - default/wait-failed-machine-status: ["2m", "10s"] - default/wait-infra-subnets: ["5m", "30s"] - default/wait-machine-pool-nodes: ["40m", "10s"] - default/wait-machine-pool-upgrade: ["50m", "10s"] - default/wait-create-identity: ["1m", "10s"] - default/wait-job: ["10m", "10s"] - default/wait-deployment-ready: ["5m", "10s"] - default/wait-loadbalancer-ready: ["5m", "30s"] + default/wait-cluster: [ "35m", "10s" ] + default/wait-control-plane: [ "35m", "10s" ] + default/wait-worker-nodes: [ "20m", "10s" ] + conformance/wait-control-plane: [ "35m", "10s" ] + conformance/wait-worker-nodes: [ "35m", "10s" ] + default/wait-controllers: [ "5m", "10s" ] + default/wait-delete-cluster: [ "20m", "10s" ] + default/wait-machine-upgrade: [ "35m", "10s" ] + default/wait-contolplane-upgrade: [ "40m", "10s" ] + default/wait-machine-status: [ "25m", "10s" ] + default/wait-failed-machine-status: [ "2m", "10s" ] + default/wait-infra-subnets: [ "5m", "30s" ] + default/wait-machine-pool-nodes: [ "40m", "10s" ] + default/wait-machine-pool-upgrade: [ "50m", "10s" ] + default/wait-create-identity: [ "1m", "10s" ] + default/wait-job: [ "10m", "10s" ] + default/wait-deployment-ready: [ "5m", "10s" ] + default/wait-loadbalancer-ready: [ "5m", "30s" ] diff --git a/test/e2e/data/shared/v1beta1/metadata-aws.yaml b/test/e2e/data/shared/v1beta1/metadata-aws.yaml deleted file mode 100644 index 8b0680e2..00000000 --- a/test/e2e/data/shared/v1beta1/metadata-aws.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# maps release series of major.minor to cluster-api contract version, -# update this file only when you update the version for cluster-api -# CoreProvider and docker InfrastructureProvider in test/e2e/config/k3s-docker.yaml -apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 -kind: Metadata -releaseSeries: - - major: 2 - minor: 6 - contract: v1beta1 diff --git a/test/e2e/data/shared/v1beta1/metadata.yaml b/test/e2e/data/shared/v1beta1/metadata.yaml index 92b9968a..fb94faeb 100644 --- a/test/e2e/data/shared/v1beta1/metadata.yaml +++ b/test/e2e/data/shared/v1beta1/metadata.yaml @@ -1,9 +1,9 @@ -# maps release series of major.minor to cluster-api contract version, -# update this file only when you update the version for cluster-api -# CoreProvider and docker InfrastructureProvider in test/e2e/config/k3s-docker.yaml apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 kind: Metadata releaseSeries: - major: 1 - minor: 6 + minor: 7 contract: v1beta1 + - major: 1 + minor: 6 + contract: v1beta1 \ No newline at end of file diff --git a/test/e2e/data/shared/v1beta1_aws/metadata.yaml b/test/e2e/data/shared/v1beta1_aws/metadata.yaml new file mode 100644 index 00000000..2500dcce --- /dev/null +++ b/test/e2e/data/shared/v1beta1_aws/metadata.yaml @@ -0,0 +1,5 @@ +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +releaseSeries: + - major: 2 + minor: 6 + contract: v1beta1 \ No newline at end of file diff --git a/tilt-provider.yaml b/tilt-provider.yaml new file mode 100644 index 00000000..d715c79b --- /dev/null +++ b/tilt-provider.yaml @@ -0,0 +1,24 @@ +- name: ck8s-bootstrap + config: + context: bootstrap + image: ghcr.io/canonical/cluster-api-k8s/bootstrap-controller + live_reload_deps: + - main.go + - api + - controllers + - ../go.mod + - ../go.sum + - ../pkg + label: CABPCK +- name: ck8s-control-plane + config: + context: controlplane + image: ghcr.io/canonical/cluster-api-k8s/controlplane-controller + live_reload_deps: + - main.go + - api + - controllers + - ../go.mod + - ../go.sum + - ../pkg + label: CACPPCK From 817c4a1a50d69d7c8819d626c110a0278c52c48b Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Fri, 23 Aug 2024 21:18:24 -0400 Subject: [PATCH 16/79] revert add aws pkg --- go.mod | 52 +++++++++++++++------------------- go.sum | 57 -------------------------------------- test/e2e/e2e_suite_test.go | 2 -- 3 files changed, 23 insertions(+), 88 deletions(-) diff --git a/go.mod b/go.mod index 19c59ca1..c52d39e8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22.6 require ( github.com/canonical/k8s-snap-api v1.0.11 github.com/go-logr/logr v1.4.1 - github.com/google/uuid v1.6.0 + github.com/google/uuid v1.4.0 github.com/onsi/ginkgo v1.16.5 github.com/onsi/ginkgo/v2 v2.17.1 github.com/onsi/gomega v1.32.0 @@ -15,7 +15,7 @@ require ( k8s.io/apimachinery v0.29.3 k8s.io/apiserver v0.29.3 k8s.io/client-go v0.29.3 - k8s.io/klog/v2 v2.120.1 + k8s.io/klog/v2 v2.110.1 k8s.io/utils v0.0.0-20240102154912-e7106e64919e sigs.k8s.io/cluster-api v1.7.1 sigs.k8s.io/cluster-api/test v1.7.1 @@ -24,32 +24,26 @@ require ( ) require ( - github.com/aws/aws-sdk-go v1.51.17 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - sigs.k8s.io/cluster-api-provider-aws/v2 v2.6.1 // indirect -) - -require ( - github.com/BurntSushi/toml v1.2.1 // indirect + github.com/BurntSushi/toml v1.0.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.2.1 // indirect + github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Microsoft/go-winio v0.5.0 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/adrg/xdg v0.4.0 // indirect - github.com/alessio/shellescape v1.4.2 // indirect + github.com/alessio/shellescape v1.4.1 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect - github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect + github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.5.0 // indirect github.com/docker/docker v25.0.6+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect - github.com/docker/go-units v0.5.0 // indirect + github.com/docker/go-units v0.4.0 // indirect github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect @@ -66,7 +60,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/cel-go v0.17.8 // indirect + github.com/google/cel-go v0.17.7 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-github/v53 v53.2.0 // indirect @@ -75,7 +69,7 @@ require ( github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/huandu/xstrings v1.4.0 // indirect + github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -94,13 +88,13 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect - github.com/prometheus/client_golang v1.19.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.52.2 // indirect - github.com/prometheus/procfs v0.13.0 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect @@ -108,7 +102,7 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cobra v1.8.0 // indirect - github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.18.2 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect @@ -119,17 +113,17 @@ require ( go.opentelemetry.io/otel/trace v1.22.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.22.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/mod v0.19.0 - golang.org/x/net v0.24.0 // indirect - golang.org/x/oauth2 v0.19.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/term v0.19.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.18.0 // indirect + golang.org/x/tools v0.17.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect @@ -141,7 +135,7 @@ require ( k8s.io/apiextensions-apiserver v0.29.3 // indirect k8s.io/cluster-bootstrap v0.29.3 // indirect k8s.io/component-base v0.29.3 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kind v0.22.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index 660df96f..33ea1288 100644 --- a/go.sum +++ b/go.sum @@ -2,16 +2,12 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= -github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU= @@ -22,18 +18,12 @@ github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= -github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= -github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/aws/aws-sdk-go v1.51.17 h1:Cfa40lCdjv9OxC3X1Ks3a6O1Tu3gOANSyKHOSw/zuWU= -github.com/aws/aws-sdk-go v1.51.17/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -43,11 +33,8 @@ github.com/canonical/k8s-snap-api v1.0.11 h1:nGtwrUQBLiaL3HUXFx2gb4kq6qVpl2yNwMw github.com/canonical/k8s-snap-api v1.0.11/go.mod h1:LDPoIYCeYnfgOFrwVPJ/4edGU264w7BB7g0GsVi36AY= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -80,8 +67,6 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 h1:7QPwrLT79GlD5sizHf27aoY2RTvw62mO6x7mxkScNk0= github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46/go.mod h1:esf2rsHFNlZlxsqsZDojNBcnNs5REqIvRrWRHqX0vEU= github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= @@ -135,8 +120,6 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/cel-go v0.17.7 h1:6ebJFzu1xO2n7TLtN+UBqShGBhlD85bhvglh5DpcfqQ= github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= -github.com/google/cel-go v0.17.8 h1:j9m730pMZt1Fc4oKhCLUHfjj6527LuhYcYw0Rl8gqto= -github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -162,8 +145,6 @@ github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2/go.mod h1:Tv1PlzqC github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -175,8 +156,6 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= -github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= @@ -185,9 +164,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -251,8 +227,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -266,20 +240,12 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= -github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= -github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= -github.com/prometheus/common v0.52.2 h1:LW8Vk7BccEdONfrJBDffQGRtpSzi5CQaRZGtboOO2ck= -github.com/prometheus/common v0.52.2/go.mod h1:lrWtQx+iDfn2mbH5GUzlH9TSHyfZpHkSiG1W7y3sF2Q= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o= -github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -293,7 +259,6 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= @@ -306,8 +271,6 @@ github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace h1:9PNP1jnUjRhfmGMlkXHjYPishpcw4jpSt/V/xYY3FMA= -github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= @@ -325,7 +288,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= @@ -372,8 +334,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -392,12 +352,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= -golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg= -golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -428,15 +384,11 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -454,8 +406,6 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= -golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -521,21 +471,14 @@ k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo= k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio= k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= -k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 h1:/U5vjBbQn3RChhv7P11uhYvCSm5G2GaIi5AIGBS6r4c= sigs.k8s.io/cluster-api v1.7.1 h1:JkMAbAMzBM+WBHxXLTJXTiCisv1PAaHRzld/3qrmLYY= sigs.k8s.io/cluster-api v1.7.1/go.mod h1:V9ZhKLvQtsDODwjXOKgbitjyCmC71yMBwDcMyNNIov0= -sigs.k8s.io/cluster-api-provider-aws/v2 v2.6.1 h1:vbZUYEB7OfPlfHk6wis+UrvRLTqv5F4Nrjl2WDJ1kiw= -sigs.k8s.io/cluster-api-provider-aws/v2 v2.6.1/go.mod h1:1aq1EZbirRW6NC2gYUFCc7cVFwX9PM/vDvoU+2oGPuw= sigs.k8s.io/cluster-api/test v1.7.1 h1:QDru2586ZjIFBTW1Z7VVXVtauzR/yANm4tglUNLm9iE= sigs.k8s.io/cluster-api/test v1.7.1/go.mod h1:yG0g5Mdq73fMn9JP4akgRQPSne973L+Qx6iVH+LjtSM= sigs.k8s.io/controller-runtime v0.17.3 h1:65QmN7r3FWgTxDMz9fvGnO1kbf2nu+acg9p2R9oYYYk= diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 509d7435..9dd78e93 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -39,7 +39,6 @@ import ( "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" dockerinfrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1" - awsinfrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" ctrl "sigs.k8s.io/controller-runtime" bootstrapv1 "github.com/canonical/cluster-api-k8s/bootstrap/api/v1beta2" @@ -197,7 +196,6 @@ func initScheme() *runtime.Scheme { Expect(controlplanev1.AddToScheme(sc)).To(Succeed()) Expect(bootstrapv1.AddToScheme(sc)).To(Succeed()) Expect(dockerinfrav1.AddToScheme(sc)).To(Succeed()) - Expect(awsinfrav1.AddToScheme(sc)).To(Succeed()) return sc } From 3bf3bc7a4eda6b8b34cfee515ad532fbe91a742f Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Fri, 23 Aug 2024 21:20:50 -0400 Subject: [PATCH 17/79] newline at eof --- test/e2e/data/shared/v1beta1/metadata.yaml | 2 +- test/e2e/data/shared/v1beta1_aws/metadata.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/data/shared/v1beta1/metadata.yaml b/test/e2e/data/shared/v1beta1/metadata.yaml index fb94faeb..c1a68366 100644 --- a/test/e2e/data/shared/v1beta1/metadata.yaml +++ b/test/e2e/data/shared/v1beta1/metadata.yaml @@ -6,4 +6,4 @@ releaseSeries: contract: v1beta1 - major: 1 minor: 6 - contract: v1beta1 \ No newline at end of file + contract: v1beta1 diff --git a/test/e2e/data/shared/v1beta1_aws/metadata.yaml b/test/e2e/data/shared/v1beta1_aws/metadata.yaml index 2500dcce..8e288cef 100644 --- a/test/e2e/data/shared/v1beta1_aws/metadata.yaml +++ b/test/e2e/data/shared/v1beta1_aws/metadata.yaml @@ -2,4 +2,4 @@ apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 releaseSeries: - major: 2 minor: 6 - contract: v1beta1 \ No newline at end of file + contract: v1beta1 From 0a3e71022f77e0526b9875f8b339ff2a839493eb Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 27 Aug 2024 20:11:07 -0400 Subject: [PATCH 18/79] add info to readme how to use existing cluster when running the tests, how to cleanup between test runs --- test/e2e/README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/e2e/README.md b/test/e2e/README.md index 849be79c..141a19d8 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -21,6 +21,50 @@ To run a specific e2e test, such as `[PR-Blocking]`, use the `GINKGO_FOCUS` envi make GINKGO_FOCUS="\\[PR-Blocking\\]" test-e2e # only run e2e test with `[PR-Blocking]` in its spec name ``` +### Use an existing cluster as the management cluster + +This is useful if you want to use a cluster managed by Tilt. + +```shell +make USE_EXISTING_CLUSTER=true test-e2e +``` + +### Cleaning up after an e2e test + +The test framework tries it's best to cleanup resources after a test suite, but it is possible that +cloud resources are left over. This can be very problematic especially if you run the tests multiple times +while iterating on development (see [Cluster API Book - Tear down](https://cluster-api.sigs.k8s.io/developer/e2e#tear-down)). + +You can use a tool like [aws-nuke](https://github.com/rebuy-de/aws-nuke) to cleanup your AWS account after a test. Here is a config. you can use that should cover most resources: + +```yaml +regions: + - us-east-2 + +account-blocklist: + - "" + +accounts: + "": {} + +resource-types: + targets: + - EC2Instance + - EC2SecurityGroup + - EC2Volume + - EC2InternetGateway + - EC2NATGateway + - EC2RouteTable + - EC2Subnet + - EC2VPC + - EC2VPCEndpoint + - EC2VPCEndpointServiceConfiguration + - EC2ElasticIP + - EC2NetworkInterface + - ELBv2 + - ELBv2TargetGroup +``` + ## Develop an e2e test Refer to [Developing E2E tests](https://cluster-api.sigs.k8s.io/developer/e2e) for a complete guide for developing e2e tests. From 9255fd557cbb7c82e72be5b9f3f1689aedd95069 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 27 Aug 2024 20:12:01 -0400 Subject: [PATCH 19/79] add region env. var --- test/e2e/config/ck8s-aws.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index eed9e2eb..d0226900 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -97,6 +97,7 @@ variables: AWS_AMI_ID: "ami-05145146e3a9db6f3" AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE: 16 AWS_NODE_ROOT_VOLUME_SIZE: 16 + AWS_REGION: "us-east-2" AWS_CCM_IMAGE: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.30.0" # https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml#L203C1-L205C27 EXP_CLUSTER_RESOURCE_SET: "true" From 98f590ac49a7ba5360c4101cf0526ae3c84a0742 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 27 Aug 2024 20:38:06 -0400 Subject: [PATCH 20/79] explain how to run the tests with tilt --- test/e2e/README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test/e2e/README.md b/test/e2e/README.md index 141a19d8..f092ebfd 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -29,6 +29,45 @@ This is useful if you want to use a cluster managed by Tilt. make USE_EXISTING_CLUSTER=true test-e2e ``` +### Running the tests with Tilt + +This section explains how to run the E2E tests on AWS using a management cluster run by Tilt. + +This section assumes you have *kind* and *Docker* installed. (See [Prerequisites](https://cluster-api.sigs.k8s.io/developer/tilt#prerequisites)) + +First, clone the upstream cluster-api and cluster-api-provider-aws repositories. +```shell +git clone https://github.com/kubernetes-sigs/cluster-api.git +git clone https://github.com/kubernetes-sigs/cluster-api-provider-aws.git +``` + +Next, you need to create a `tilt-settings.yaml` file inside the `cluster-api` directory. +The kustomize_substitutions you provide here are automatically applied to the *management cluster*. +```shell +default_registry: "ghcr.io/canonical/cluster-api-k8s" +provider_repos: +- ../cluster-api-k8s +- ../cluster-api-provider-aws +enable_providers: +- aws +- ck8s-bootstrap +- ck8s-control-plane +kustomize_substitutions: + EXP_CLUSTER_RESOURCE_SET: "true" # todo(eac): revisit these exp. features + EXP_MACHINE_POOL: "true" + CAPA_EKS_IAM: "false" + CAPA_EKS_ADD_ROLES: "false" + AWS_B64ENCODED_CREDENTIALS: "" +``` + +Tilt will know how to run the aws provider controllers because the `cluster-api-provider-aws` repository has a `tilt-provider.json` file at it's root. Canonical Kubernetes also provides this file at the root of the repository. The CK8s provider names, ck8s-bootstrap and ck8s-control-plane, are defined in CK8's `tilt-provider.yaml` file. + +Next, you have to customize the variables that will be substituted into the cluster templates applied by the tests (these are under `test/e2e/data/infrastructure-aws`). You can customize the variables in the `test/e2e/config/ck8s-aws.yaml` file under the `variables` key. + +Finally, in one terminal, go into the `cluster-api` directory and run `make tilt-up`. You should see a kind cluster be created, and finally a message indicating that Tilt is available at a certain address. + +In a second terminal in the `cluster-api-k8s` directory, run `make USE_EXISTING_CLUSTER=true test-e2e`. + ### Cleaning up after an e2e test The test framework tries it's best to cleanup resources after a test suite, but it is possible that From 6d66f7b4c75f7bb5d6b8ae04b1bdc576ecbd4f86 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 29 Aug 2024 18:50:15 -0400 Subject: [PATCH 21/79] add the ability to run tests on aws Adds ability to configure infrastructure provider through the Makefile. Change from hardcoded infra provider strings to clusterctl.DefaultInfraProvider Adds templates for kcp remediation and md remediation, though these do not pass yet. Disables kubelet and containerd services in case the cluster is being deployed on a kubeadm AMI. --- .github/workflows/e2e.yaml | 18 +- Makefile | 7 +- hack/install-clusterctl.sh | 17 + hack/install-clusterctlawsadm.sh | 17 + hack/write-provider-config.sh | 24 ++ test/e2e/cluster_upgrade_test.go | 3 +- test/e2e/config/ck8s-aws.yaml | 2 + test/e2e/create_test.go | 2 +- .../cluster-template-kcp-remediation.yaml | 387 ++++++++++++++++++ .../cluster-template-md-remediation.yaml | 363 ++++++++++++++++ .../infrastructure-aws/cluster-template.yaml | 14 +- test/e2e/kcp_remediation_test.go | 3 +- test/e2e/md_remediation_test.go | 2 +- test/e2e/node_scale_test.go | 2 +- 14 files changed, 851 insertions(+), 10 deletions(-) create mode 100755 hack/install-clusterctl.sh create mode 100644 hack/install-clusterctlawsadm.sh create mode 100644 hack/write-provider-config.sh create mode 100644 test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml create mode 100644 test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2fbbd3a3..2a128c3c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -59,9 +59,12 @@ jobs: needs: build-e2e-images strategy: matrix: + infra-provider: + #- docker + - aws ginkgo_focus: - - "KCP remediation" - - "MachineDeployment remediation" + #- "KCP remediation" + #- "MachineDeployment remediation" - "Workload cluster creation" - "Workload cluster scaling" - "Workload cluster upgrade" @@ -85,6 +88,8 @@ jobs: sudo apt install make sudo apt install docker-buildx sudo snap install kubectl --classic --channel=1.30/stable + sudo ./hack/install-clusterctlawsadm.sh + sudo ./hack/install-clusterctl.sh - name: Download artifacts uses: actions/download-artifact@v4 with: @@ -109,4 +114,13 @@ jobs: sudo sysctl fs.inotify.max_user_instances=8192 - name: Run e2e tests run: | + mkdir -p ~/.config/cluster-api + mkdir ~/.kube + export AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile) + sudo ./hack/write-provider-config.sh ~/.config/cluster-api/clusterctl.yaml v0.1.2 + sudo docker run --name management-cluster --network kind --rm --detach --privileged + sudo docker exec management-cluster k8s bootstrap + sleep 1 + sudo docker exec management-cluster k8s config > ~/.kube/config + sed -i 's/E2E_INFRA ?= docker/E2E_INFRA ?= ${{ matrix.infra-provider }}/' ./Makefile sudo GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e diff --git a/Makefile b/Makefile index ecf8229f..bce01e26 100644 --- a/Makefile +++ b/Makefile @@ -99,7 +99,8 @@ GINKGO_NODES ?= 1 # GINKGO_NODES is the number of parallel nodes to run GINKGO_TIMEOUT ?= 2h GINKGO_POLL_PROGRESS_AFTER ?= 60m GINKGO_POLL_PROGRESS_INTERVAL ?= 5m -E2E_CONF_FILE ?= $(TEST_DIR)/e2e/config/ck8s-aws.yaml +E2E_INFRA ?= aws +E2E_CONF_FILE ?= $(TEST_DIR)/e2e/config/ck8s-$(E2E_INFRA).yaml SKIP_RESOURCE_CLEANUP ?= false USE_EXISTING_CLUSTER ?= false GINKGO_NOCOLOR ?= false @@ -407,3 +408,7 @@ $(CONTROLLER_GEN): ## Build controller-gen from tools folder. $(CONVERSION_GEN): ## Build conversion-gen from tools folder. GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) k8s.io/code-generator/cmd/conversion-gen $(CONVERSION_GEN_BIN) $(CONVERSION_GEN_VER) + +.PHONY: nuke +nuke: + aws-nuke -c ~/.config/aws-nuke/nuke-config.yaml --no-dry-run diff --git a/hack/install-clusterctl.sh b/hack/install-clusterctl.sh new file mode 100755 index 00000000..5be8b88a --- /dev/null +++ b/hack/install-clusterctl.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Description: +# Installs clusterctl to /usr/local/bin +# +# Usage: +# $ install-clusterctl.sh + +set -o errexit +set -o nounset +set -o pipefail + +curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.1/clusterctl-linux-amd64 -o clusterctl +chmod +x ./clusterctl +sudo mv ./clusterctl /usr/local/bin + +clusterctl version diff --git a/hack/install-clusterctlawsadm.sh b/hack/install-clusterctlawsadm.sh new file mode 100644 index 00000000..6dd594c4 --- /dev/null +++ b/hack/install-clusterctlawsadm.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Description: +# Installs clusterawsadm to /usr/local/bin +# +# Usage: +# $ install-clusterawsadm.sh + +set -o errexit +set -o nounset +set -o pipefail + +curl -L https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v0.0.0/clusterawsadm-linux-amd64 -o clusterawsadm +chmod +x ./clusterawsadm +sudo mv ./clusterawsadm /usr/local/bin + +clusterawsadm version diff --git a/hack/write-provider-config.sh b/hack/write-provider-config.sh new file mode 100644 index 00000000..495ea3f6 --- /dev/null +++ b/hack/write-provider-config.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Description: +# Creates a clusterctl configuration file +# +# Usage: +# $ write-clusterctl-config.sh $output-file $version + +set -o errexit +set -o nounset +set -o pipefail + +$OUTPUT_FILE=$1 +$VERSION=$2 + +cat << 'EOF' > "$output_file" +providers: + - name: "ck8s" + url: "https://github.com/canonical/cluster-api-k8s/releases/download/${VERSION}/bootstrap-components.yaml" + type: "BootstrapProvider" + - name: "ck8s" + url: "https://github.com/canonical/cluster-api-k8s/releases/download/${VERSION}/control-plane-components.yaml" + type: "ControlPlaneProvider" +EOF diff --git a/test/e2e/cluster_upgrade_test.go b/test/e2e/cluster_upgrade_test.go index 80dc7052..5b9841c1 100644 --- a/test/e2e/cluster_upgrade_test.go +++ b/test/e2e/cluster_upgrade_test.go @@ -22,6 +22,7 @@ package e2e import ( . "github.com/onsi/ginkgo/v2" "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" ) var _ = Describe("Workload cluster upgrade [CK8s-Upgrade]", func() { @@ -33,7 +34,7 @@ var _ = Describe("Workload cluster upgrade [CK8s-Upgrade]", func() { BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - InfrastructureProvider: ptr.To("docker"), + InfrastructureProvider: ptr.To(clusterctl.DefaultInfrastructureProvider), ControlPlaneMachineCount: ptr.To[int64](3), WorkerMachineCount: ptr.To[int64](1), } diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index d0226900..c80c540b 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -55,6 +55,8 @@ providers: new: "imagePullPolicy: IfNotPresent" files: - sourcePath: "../data/infrastructure-aws/cluster-template.yaml" + - sourcePath: "../data/infrastructure-aws/cluster-template-md-remediation.yaml" + - sourcePath: "../data/infrastructure-aws/cluster-template-kcp-remediation.yaml" - name: ck8s type: BootstrapProvider versions: diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go index 63497e17..41c0e762 100644 --- a/test/e2e/create_test.go +++ b/test/e2e/create_test.go @@ -48,7 +48,7 @@ var _ = Describe("Workload cluster creation", func() { Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion)) clusterName = fmt.Sprintf("capick8s-create-%s", util.RandomString(6)) - infrastructureProvider = "aws" + infrastructureProvider = clusterctl.DefaultInfrastructureProvider // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) diff --git a/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml new file mode 100644 index 00000000..3862314e --- /dev/null +++ b/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml @@ -0,0 +1,387 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + labels: + ccm: external +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.1.0.0/16 + services: + cidrBlocks: + - 10.152.183.0/24 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSCluster +metadata: + name: ${CLUSTER_NAME} +spec: + region: ${AWS_REGION} + sshKeyName: ${AWS_SSH_KEY_NAME} + bastion: + enabled: ${AWS_CREATE_BASTION} + controlPlaneLoadBalancer: + healthCheckProtocol: TCP + network: + cni: + cniIngressRules: + - description: microcluster + protocol: tcp + toPort: 2380 +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: ${CLUSTER_NAME}-control-plane + spec: + nodeName: "{{ ds.meta_data.local_hostname }}" + preRunCommands: + - systemctl stop kubelet || true + - systemctl disable kubelet || true + - systemctl stop containerd || true + - systemctl disable containerd || true + - ./wait-signal.sh "${TOKEN}" "${SERVER}" "${NAMESPACE}" + files: + # note(ben): This is only required as long as k8s does not have a stable release. + - path: /capi/scripts/install.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + snap install k8s --classic --channel=latest/edge/etienne-delete-me + + - path: /wait-signal.sh + content: | + #!/bin/bash + + set -o errexit + set -o pipefail + + echo "Waiting for signal..." + + TOKEN=$1 + SERVER=$2 + NAMESPACE=$3 + + while true; + do + sleep 1s + + signal=$(curl -k -s --header "Authorization: Bearer $TOKEN" $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test | jq -r .data.signal?) + echo "signal $signal" + + if [ "$signal" == "pass" ]; then + curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test + exit 0 + fi + done + permissions: "0777" + owner: root:root + controlPlane: + cloudProvider: external + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + ami: + id: ${AWS_AMI_ID} + iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_CONTROL_PLANE_INSTANCE_TYPE} + publicIP: ${AWS_PUBLIC_IP} + sshKeyName: ${AWS_SSH_KEY_NAME} + rootVolume: + size: ${AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-worker-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + template: + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: CK8sConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + ami: + id: ${AWS_AMI_ID} + iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_NODE_INSTANCE_TYPE} + publicIP: ${AWS_PUBLIC_IP} + sshKeyName: ${AWS_SSH_KEY_NAME} + rootVolume: + size: ${AWS_NODE_ROOT_VOLUME_SIZE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + nodeName: "{{ ds.meta_data.local_hostname }}" + preRunCommands: + - systemctl stop kubelet || true + - systemctl disable kubelet || true + - systemctl stop containerd || true + - systemctl disable containerd || true + files: + # note(ben): This is only required as long as k8s does not have a stable release. + - path: /capi/scripts/install.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + snap install k8s --classic --channel=latest/edge/etienne-delete-me +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: ${AWS_CCM_IMAGE} + args: + - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + - --configure-cloud-routes=false + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-mhc-0 + namespace: ${NAMESPACE} +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + nodeStartupTimeout: 30s + selector: + matchLabels: + cluster.x-k8s.io/control-plane: "" + mhc-test: fail + unhealthyConditions: + - status: "False" + timeout: 10s + type: e2e.remediation.condition diff --git a/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml new file mode 100644 index 00000000..c2211828 --- /dev/null +++ b/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml @@ -0,0 +1,363 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + labels: + ccm: external +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.1.0.0/16 + services: + cidrBlocks: + - 10.152.183.0/24 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSCluster +metadata: + name: ${CLUSTER_NAME} +spec: + region: ${AWS_REGION} + sshKeyName: ${AWS_SSH_KEY_NAME} + bastion: + enabled: ${AWS_CREATE_BASTION} + controlPlaneLoadBalancer: + healthCheckProtocol: TCP + network: + cni: + cniIngressRules: + - description: microcluster + protocol: tcp + toPort: 2380 +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: ${CLUSTER_NAME}-control-plane + spec: + nodeName: "{{ ds.meta_data.local_hostname }}" + preRunCommands: + - systemctl stop kubelet || true + - systemctl disable kubelet || true + - systemctl stop containerd || true + - systemctl disable containerd || true + files: + # note(ben): This is only required as long as k8s does not have a stable release. + - path: /capi/scripts/install.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + snap install k8s --classic --channel=latest/edge/etienne-delete-me + controlPlane: + cloudProvider: external + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + ami: + id: ${AWS_AMI_ID} + iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_CONTROL_PLANE_INSTANCE_TYPE} + publicIP: ${AWS_PUBLIC_IP} + sshKeyName: ${AWS_SSH_KEY_NAME} + rootVolume: + size: ${AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-worker-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-worker-md-0 + template: + metadata: + labels: + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-worker-md-0 + e2e.remediation.label: "" + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: CK8sConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + ami: + id: ${AWS_AMI_ID} + iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_NODE_INSTANCE_TYPE} + publicIP: ${AWS_PUBLIC_IP} + sshKeyName: ${AWS_SSH_KEY_NAME} + rootVolume: + size: ${AWS_NODE_ROOT_VOLUME_SIZE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + nodeName: "{{ ds.meta_data.local_hostname }}" + preRunCommands: + - systemctl stop kubelet || true + - systemctl disable kubelet || true + - systemctl stop containerd || true + - systemctl disable containerd || true + files: + # note(ben): This is only required as long as k8s does not have a stable release. + - path: /capi/scripts/install.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + snap install k8s --classic --channel=latest/edge/etienne-delete-me +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: ${AWS_CCM_IMAGE} + args: + - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + - --configure-cloud-routes=false + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + name: cloud-controller-manager-addon + namespace: default +--- +# MachineHealthCheck object with +# - a selector that targets all the machines with label e2e.remediation.label="" +# - unhealthyConditions triggering remediation after 10s the condition is set +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: "${CLUSTER_NAME}-mhc-0" +spec: + clusterName: "${CLUSTER_NAME}" + maxUnhealthy: 100% + selector: + matchLabels: + e2e.remediation.label: "" + unhealthyConditions: + - type: e2e.remediation.condition + status: "False" + timeout: 10s diff --git a/test/e2e/data/infrastructure-aws/cluster-template.yaml b/test/e2e/data/infrastructure-aws/cluster-template.yaml index 66f83b7c..057ece5c 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template.yaml @@ -52,6 +52,11 @@ spec: name: ${CLUSTER_NAME}-control-plane spec: nodeName: "{{ ds.meta_data.local_hostname }}" + preRunCommands: + - systemctl stop kubelet || true + - systemctl disable kubelet || true + - systemctl stop containerd || true + - systemctl disable containerd || true files: # note(ben): This is only required as long as k8s does not have a stable release. - path: /capi/scripts/install.sh @@ -59,7 +64,7 @@ spec: owner: "root:root" content: | #!/bin/bash -xe - snap install k8s --classic --edge + snap install k8s --classic --channel=latest/edge/etienne-delete-me controlPlane: cloudProvider: external replicas: ${CONTROL_PLANE_MACHINE_COUNT} @@ -129,6 +134,11 @@ spec: template: spec: nodeName: "{{ ds.meta_data.local_hostname }}" + preRunCommands: + - systemctl stop kubelet || true + - systemctl disable kubelet || true + - systemctl stop containerd || true + - systemctl disable containerd || true files: # note(ben): This is only required as long as k8s does not have a stable release. - path: /capi/scripts/install.sh @@ -136,7 +146,7 @@ spec: owner: "root:root" content: | #!/bin/bash -xe - snap install k8s --classic --edge + snap install k8s --classic --channel=latest/edge/etienne-delete-me --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet diff --git a/test/e2e/kcp_remediation_test.go b/test/e2e/kcp_remediation_test.go index 881f1907..439a8519 100644 --- a/test/e2e/kcp_remediation_test.go +++ b/test/e2e/kcp_remediation_test.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo/v2" "k8s.io/utils/ptr" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" ) var _ = Describe("When testing KCP remediation", func() { @@ -33,6 +34,6 @@ var _ = Describe("When testing KCP remediation", func() { BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - InfrastructureProvider: ptr.To("docker")} + InfrastructureProvider: ptr.To(clusterctl.DefaultInfrastructureProvider)} }) }) diff --git a/test/e2e/md_remediation_test.go b/test/e2e/md_remediation_test.go index 4f707ba2..c50852a1 100644 --- a/test/e2e/md_remediation_test.go +++ b/test/e2e/md_remediation_test.go @@ -49,7 +49,7 @@ var _ = Describe("When testing MachineDeployment remediation", func() { Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion)) clusterName = fmt.Sprintf("capick8s-md-remediation-%s", util.RandomString(6)) - infrastructureProvider = "docker" + infrastructureProvider = clusterctl.DefaultInfrastructureProvider // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) diff --git a/test/e2e/node_scale_test.go b/test/e2e/node_scale_test.go index e295b450..4652e2f5 100644 --- a/test/e2e/node_scale_test.go +++ b/test/e2e/node_scale_test.go @@ -48,7 +48,7 @@ var _ = Describe("Workload cluster scaling", func() { Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion)) clusterName = fmt.Sprintf("capick8s-node-scale-%s", util.RandomString(6)) - infrastructureProvider = "docker" + infrastructureProvider = clusterctl.DefaultInfrastructureProvider // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) From c21680b72eb3d964e3c2a329bfb5d372de7a85ba Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 3 Sep 2024 16:14:20 -0400 Subject: [PATCH 22/79] Add running cluster in CI --- hack/ci-e2e-tests.sh | 85 +++++++++++++++++++++++++++++++++++ hack/write-provider-config.sh | 2 +- 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100755 hack/ci-e2e-tests.sh diff --git a/hack/ci-e2e-tests.sh b/hack/ci-e2e-tests.sh new file mode 100755 index 00000000..05f3f077 --- /dev/null +++ b/hack/ci-e2e-tests.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +set -xe + +SKIP_CLEANUP=${1:-true} + +LXD_CHANNEL="5.21/stable" +LXC_IMAGE="ubuntu:20.04" +K8S_PROFILE_URL="https://raw.githubusercontent.com/canonical/k8s-snap/main/tests/integration/lxd-profile.yaml" +K8S_PROFILE_PATH="k8s.profile" +CONTAINER_NAME="k8s-test" +USER_CREDENTIALS_PATH="/home/user/.creds" + +# Install LXD snap +function install_lxd { + sudo snap install lxd --channel=$LXD_CHANNEL + sudo lxd init --auto + sudo usermod --append --groups lxd $USER +} + +# Create or ensure the k8s profile exists +function setup_lxd_profile { + lxc profile create k8s || true + wget -q $K8S_PROFILE_URL -O $K8S_PROFILE_PATH + cat $K8S_PROFILE_PATH | lxc profile edit k8s + rm -f $K8S_PROFILE_PATH +} + +# Setup and configure the container +function setup_container { + lxc launch $LXC_IMAGE $CONTAINER_NAME -p default -p k8s + sleep 3 # Wait for the container to be up and running + lxc exec $CONTAINER_NAME -- bash -c "apt update && apt install -y snapd" + sleep 1 + lxc exec $CONTAINER_NAME -- bash -c "systemctl start snapd" +} + +# Main installation and configuration +function setup_management_cluster { + lxc exec $CONTAINER_NAME -- bash -c "snap install k8s --classic --edge" + lxc exec $CONTAINER_NAME -- bash -c "snap install go --classic" + sleep 1 + lxc exec $CONTAINER_NAME -- bash -c "mkdir -p /root/.kube" + lxc exec $CONTAINER_NAME -- bash -c "sudo k8s bootstrap" + lxc exec $CONTAINER_NAME -- bash -c "sudo k8s status --wait-ready" + lxc exec $CONTAINER_NAME -- bash -c "sudo k8s config > /root/.kube/config" +} + +# Transfer and execute scripts +function install_tools { + for script in install-clusterctl.sh install-clusterctlawsadm.sh install-aws-nuke.sh write-provider-config.sh; do + lxc file push ./hack/$script $CONTAINER_NAME/root/$script + if [[ ! $script == "write-provider-config.sh" ]]; then + lxc exec $CONTAINER_NAME -- bash -c "chmod +x /root/$script && /root/$script" + else + lxc exec $CONTAINER_NAME -- bash -c "mkdir -p /root/.cluster-api" + lxc exec $CONTAINER_NAME -- bash -c "chmod +x /root/$script && /root/$script /root/.cluster-api/clusterctl.yaml v0.1.2" + fi + done + + lxc file push $USER_CREDENTIALS_PATH $CONTAINER_NAME/root/.creds +} + +function init_clusterctl { + lxc exec $CONTAINER_NAME -- bash -c "source /root/.creds && clusterctl init -i aws -b ck8s:v0.1.2 -c ck8s:v0.1.2 --config /root/.cluster-api/clusterctl.yaml" +} + +function cleanup { + lxc delete $CONTAINER_NAME --force +} + +function run_all { + install_lxd + setup_lxd_profile + setup_container + setup_management_cluster + install_tools + init_clusterctl + + if [[ $SKIP_CLEANUP == "false" ]]; then + cleanup + fi +} + +run_all diff --git a/hack/write-provider-config.sh b/hack/write-provider-config.sh index 495ea3f6..7f876a25 100644 --- a/hack/write-provider-config.sh +++ b/hack/write-provider-config.sh @@ -13,7 +13,7 @@ set -o pipefail $OUTPUT_FILE=$1 $VERSION=$2 -cat << 'EOF' > "$output_file" +cat << EOF > "$OUTPUT_FILE" providers: - name: "ck8s" url: "https://github.com/canonical/cluster-api-k8s/releases/download/${VERSION}/bootstrap-components.yaml" From 76dfe129d1634c96e08a46a2469db4297ba23b49 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 3 Sep 2024 16:20:00 -0400 Subject: [PATCH 23/79] reset e2e.yaml --- .github/workflows/e2e.yaml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2a128c3c..2fbbd3a3 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -59,12 +59,9 @@ jobs: needs: build-e2e-images strategy: matrix: - infra-provider: - #- docker - - aws ginkgo_focus: - #- "KCP remediation" - #- "MachineDeployment remediation" + - "KCP remediation" + - "MachineDeployment remediation" - "Workload cluster creation" - "Workload cluster scaling" - "Workload cluster upgrade" @@ -88,8 +85,6 @@ jobs: sudo apt install make sudo apt install docker-buildx sudo snap install kubectl --classic --channel=1.30/stable - sudo ./hack/install-clusterctlawsadm.sh - sudo ./hack/install-clusterctl.sh - name: Download artifacts uses: actions/download-artifact@v4 with: @@ -114,13 +109,4 @@ jobs: sudo sysctl fs.inotify.max_user_instances=8192 - name: Run e2e tests run: | - mkdir -p ~/.config/cluster-api - mkdir ~/.kube - export AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile) - sudo ./hack/write-provider-config.sh ~/.config/cluster-api/clusterctl.yaml v0.1.2 - sudo docker run --name management-cluster --network kind --rm --detach --privileged - sudo docker exec management-cluster k8s bootstrap - sleep 1 - sudo docker exec management-cluster k8s config > ~/.kube/config - sed -i 's/E2E_INFRA ?= docker/E2E_INFRA ?= ${{ matrix.infra-provider }}/' ./Makefile sudo GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e From 1100cb2604ed91db0a03889d67fc3f7508b41e55 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 3 Sep 2024 16:20:50 -0400 Subject: [PATCH 24/79] Remove new file docs/overview.md --- docs/overview.md | 143 ----------------------------------------------- 1 file changed, 143 deletions(-) delete mode 100644 docs/overview.md diff --git a/docs/overview.md b/docs/overview.md deleted file mode 100644 index cdcace65..00000000 --- a/docs/overview.md +++ /dev/null @@ -1,143 +0,0 @@ -# Cluster Provisioning with CAPI - -This guide covers how to deploy a Canonical Kubernetes multi-node cluster using Cluster API (CAPI). - -## Install `clusterctl` - -The `clusterctl` CLI tool manages the lifecycle of a Cluster API management cluster. To install it, follow the [upstream instructions]. Typically, this involves fetching the executable that matches your hardware architecture and placing it in your PATH. For example, at the time this guide was written, for `amd64` you would: - -```sh -curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.3/clusterctl-linux-amd64 -o clusterctl -sudo install -o root -g root -m 0755 clusterctl /usr/local/bin/clusterctl -``` - -### Set up a management Cluster - -The management cluster hosts the CAPI providers. You can use a Canonical Kubernetes cluster as a management cluster: - -```sh -sudo snap install k8s --classic --edge -sudo k8s bootstrap -sudo k8s status --wait-ready -mkdir -p ~/.kube/ -sudo k8s kubectl config view --raw > ~/.kube/config -``` - -When setting up the management cluster, place its kubeconfig under `~/.kube/config` so other tools such as `clusterctl` can discover and interact with it. - -### Prepare the Infrastructure Provider - -Before generating a cluster, you need to configure the infrastructure provider. Each provider has its own prerequisites. Please follow the Cluster API instructions for the additional infrastructure-specific configuration. - -#### Example Using AWS - -The AWS infrastructure provider requires the `clusterawsadm` tool to be installed: - -```sh -curl -L https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.0.2/clusterawsadm-linux-amd64 -o clusterawsadm -chmod +x clusterawsadm -sudo mv clusterawsadm /usr/local/bin -``` - -With `clusterawsadm`, you can bootstrap the AWS environment that CAPI will use. - -Start by setting up environment variables defining the AWS account to use, if these are not already defined: - -```sh -export AWS_REGION= -export AWS_ACCESS_KEY_ID= -export AWS_SECRET_ACCESS_KEY= -``` - -If you are using multi-factor authentication, you will also need: - -```sh -export AWS_SESSION_TOKEN= # If you are using Multi-Factor Auth. -``` - -The `clusterawsadm` uses these details to create a CloudFormation stack in your AWS account with the correct IAM resources: - -```sh -clusterawsadm bootstrap iam create-cloudformation-stack -``` - -The credentials should also be encoded and stored as a Kubernetes secret: - -```sh -export AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile) -``` - -### Initialize the Management Cluster - -To initialize the management cluster with the latest released version of the providers and the infrastructure of your choice: - -```sh -clusterctl init --bootstrap ck8s --control-plane ck8s -i -``` - -### Generate a Cluster Spec Manifest - -Once the bootstrap and control-plane controllers are up and running, you can apply the cluster manifests with the specifications of the cluster you want to provision. - -You can generate a cluster manifest for an infrastructure using templates provided by the Canonical Kubernetes team. The templates/ folder contains templates for common clouds. - -Ensure you have initialized the desired infrastructure provider and fetch the Canonical Kubernetes bootstrap provider repository. - -Review the list of variables needed for the cluster template: - -```sh -cd templates/ -clusterctl generate cluster k8s- --from ./templates/cluster-template-.yaml --list-variables -``` - -Set the respective environment variables by editing the rc file as needed before sourcing it. Then generate the cluster manifest: - -```sh -source ./templates/cluster-template-.rc -clusterctl generate cluster k8s- --from ./templates/cluster-template-.yaml > cluster.yaml -``` - -Each provisioned node is associated with a `K8sConfig`, through which you can set the cluster’s properties. Review the available options in the respective definitions file and edit the cluster manifest (`cluster.yaml` above) to match your needs. Note that the configuration structure is similar to that of `kubeadm` - in the `CK8sConfig`, you will find a `ClusterConfiguration` and an `InitConfiguration` section. - -### Deploy the Cluster - -To deploy the cluster, run: - -```sh -sudo k8s kubectl apply -f cluster.yaml -``` - -To see the deployed machines: - -```sh -sudo k8s kubectl get machine -``` - -After the first control plane node is provisioned, you can get the kubeconfig of the workload cluster: - -```sh -clusterctl get kubeconfig > kubeconfig -``` - -You can then see the workload nodes using: - -```sh -KUBECONFIG=./kubeconfig kubectl get node -``` - -### Delete the Cluster - -To get the list of provisioned clusters: - -```sh -sudo k8s kubectl get clusters -``` - -To delete a cluster: - -```sh -sudo k8s kubectl delete cluster -``` - - -[upstream instructions]: https://cluster-api.sigs.k8s.io/user/quick-start#install-clusterctl From 1ce27e7f7da7475a6efe77bc4404cb7233c15564 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 3 Sep 2024 21:43:11 -0400 Subject: [PATCH 25/79] Make install script more robust --- hack/aws-nuke-config.yaml | 25 ++++++++++++ hack/ci-e2e-tests.sh | 68 +++++++++++++++++++++----------- hack/install-aws-nuke.sh | 19 +++++++++ hack/install-clusterctlawsadm.sh | 2 +- hack/write-provider-config.sh | 6 +-- 5 files changed, 92 insertions(+), 28 deletions(-) create mode 100644 hack/aws-nuke-config.yaml create mode 100644 hack/install-aws-nuke.sh diff --git a/hack/aws-nuke-config.yaml b/hack/aws-nuke-config.yaml new file mode 100644 index 00000000..e3ec8d1e --- /dev/null +++ b/hack/aws-nuke-config.yaml @@ -0,0 +1,25 @@ +regions: + - us-east-2 + +account-blocklist: + - "" + +accounts: + "": {} + +resource-types: + targets: + - EC2Instance + - EC2SecurityGroup + - EC2Volume + - EC2InternetGateway + - EC2NATGateway + - EC2RouteTable + - EC2Subnet + - EC2VPC + - EC2VPCEndpoint + - EC2VPCEndpointServiceConfiguration + - EC2ElasticIP + - EC2NetworkInterface + - ELBv2 + - ELBv2TargetGroup diff --git a/hack/ci-e2e-tests.sh b/hack/ci-e2e-tests.sh index 05f3f077..6f5b41b7 100755 --- a/hack/ci-e2e-tests.sh +++ b/hack/ci-e2e-tests.sh @@ -1,21 +1,27 @@ -#!/usr/bin/env bash +#!/bin/bash set -xe -SKIP_CLEANUP=${1:-true} +readonly HACK_DIR="$(realpath $(dirname "${0}"))" -LXD_CHANNEL="5.21/stable" -LXC_IMAGE="ubuntu:20.04" -K8S_PROFILE_URL="https://raw.githubusercontent.com/canonical/k8s-snap/main/tests/integration/lxd-profile.yaml" -K8S_PROFILE_PATH="k8s.profile" -CONTAINER_NAME="k8s-test" -USER_CREDENTIALS_PATH="/home/user/.creds" +cd "$HACK_DIR" + +readonly SKIP_CLEANUP=${1:-true} +readonly +readonly LXD_CHANNEL="5.21/stable" +readonly LXC_IMAGE="ubuntu:20.04" +readonly K8S_PROFILE_URL="https://raw.githubusercontent.com/canonical/k8s-snap/main/tests/integration/lxd-profile.yaml" +readonly K8S_PROFILE_PATH="/tmp/k8s.profile" +readonly CONTAINER_NAME="k8s-test" + +# User specific paths +readonly USER_CREDENTIALS_PATH="/home/user/.creds" # Install LXD snap function install_lxd { sudo snap install lxd --channel=$LXD_CHANNEL sudo lxd init --auto - sudo usermod --append --groups lxd $USER + sudo usermod --append --groups lxd "$USER" } # Create or ensure the k8s profile exists @@ -29,7 +35,7 @@ function setup_lxd_profile { # Setup and configure the container function setup_container { lxc launch $LXC_IMAGE $CONTAINER_NAME -p default -p k8s - sleep 3 # Wait for the container to be up and running + sleep 3 # Wait for the container to be up and running lxc exec $CONTAINER_NAME -- bash -c "apt update && apt install -y snapd" sleep 1 lxc exec $CONTAINER_NAME -- bash -c "systemctl start snapd" @@ -38,8 +44,8 @@ function setup_container { # Main installation and configuration function setup_management_cluster { lxc exec $CONTAINER_NAME -- bash -c "snap install k8s --classic --edge" - lxc exec $CONTAINER_NAME -- bash -c "snap install go --classic" sleep 1 + lxc exec $CONTAINER_NAME -- bash -c "snap install go --classic" lxc exec $CONTAINER_NAME -- bash -c "mkdir -p /root/.kube" lxc exec $CONTAINER_NAME -- bash -c "sudo k8s bootstrap" lxc exec $CONTAINER_NAME -- bash -c "sudo k8s status --wait-ready" @@ -49,7 +55,7 @@ function setup_management_cluster { # Transfer and execute scripts function install_tools { for script in install-clusterctl.sh install-clusterctlawsadm.sh install-aws-nuke.sh write-provider-config.sh; do - lxc file push ./hack/$script $CONTAINER_NAME/root/$script + lxc file push ./$script $CONTAINER_NAME/root/$script if [[ ! $script == "write-provider-config.sh" ]]; then lxc exec $CONTAINER_NAME -- bash -c "chmod +x /root/$script && /root/$script" else @@ -59,27 +65,41 @@ function install_tools { done lxc file push $USER_CREDENTIALS_PATH $CONTAINER_NAME/root/.creds + lxc exec $CONTAINER_NAME -- bash -c 'echo "source /root/.creds" >/root/.bashrc' } function init_clusterctl { lxc exec $CONTAINER_NAME -- bash -c "source /root/.creds && clusterctl init -i aws -b ck8s:v0.1.2 -c ck8s:v0.1.2 --config /root/.cluster-api/clusterctl.yaml" } +function run_e2e_tests { + make GINKGO.FOCUS="Workload cluster creation" test-e2e +} + +function write_aws_nuke_config { + lxc exec $CONTAINER_NAME -- bash -c "mkdir -p /root/.aws-nuke" + lxc exec $CONTAINER_NAME -- bash -c "echo ""$AWS_NUKE_CONFIG"" > /root/.aws-nuke/config.yaml" +} + function cleanup { - lxc delete $CONTAINER_NAME --force + write_aws_nuke_config + lxc exec $CONTAINER_NAME -- bash -c "aws-nuke --config /root/.aws-nuke/config.yaml --force" + lxc delete $CONTAINER_NAME --force } -function run_all { - install_lxd - setup_lxd_profile - setup_container - setup_management_cluster - install_tools - init_clusterctl +function main { + install_lxd + setup_lxd_profile + setup_container + setup_management_cluster + install_tools + init_clusterctl + run_e2e_tests - if [[ $SKIP_CLEANUP == "false" ]]; then - cleanup - fi + if [[ $SKIP_CLEANUP == "false" ]]; then + write_aws_nuke_config + cleanup + fi } -run_all +main diff --git a/hack/install-aws-nuke.sh b/hack/install-aws-nuke.sh new file mode 100644 index 00000000..568bb7f8 --- /dev/null +++ b/hack/install-aws-nuke.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Description: +# Installs aws-nuke to /usr/local/bin +# +# Usage: +# $ install-aws-nuke.sh + +set -o errexit +set -o nounset +set -o pipefail + +curl -L https://github.com/rebuy-de/aws-nuke/releases/download/v2.25.0/aws-nuke-v2.25.0-linux-amd64.tar.gz -o aws-nuke-v2.25.0-linux-amd64.tar.gz +tar -xvf aws-nuke-v2.25.0-linux-amd64.tar.gz -C /tmp +rm aws-nuke-v2.25.0-linux-amd64.tar.gz +chmod +x /tmp/aws-nuke-v2.25.0-linux-amd64 +mv /tmp/aws-nuke-v2.25.0-linux-amd64 /usr/local/bin/aws-nuke + +aws-nuke version diff --git a/hack/install-clusterctlawsadm.sh b/hack/install-clusterctlawsadm.sh index 6dd594c4..c838f69c 100644 --- a/hack/install-clusterctlawsadm.sh +++ b/hack/install-clusterctlawsadm.sh @@ -10,7 +10,7 @@ set -o errexit set -o nounset set -o pipefail -curl -L https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v0.0.0/clusterawsadm-linux-amd64 -o clusterawsadm +curl -L https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.6.1/clusterawsadm-linux-amd64 -o clusterawsadm chmod +x ./clusterawsadm sudo mv ./clusterawsadm /usr/local/bin diff --git a/hack/write-provider-config.sh b/hack/write-provider-config.sh index 7f876a25..c396216b 100644 --- a/hack/write-provider-config.sh +++ b/hack/write-provider-config.sh @@ -10,10 +10,10 @@ set -o errexit set -o nounset set -o pipefail -$OUTPUT_FILE=$1 -$VERSION=$2 +OUTPUT_FILE="$1" +VERSION="$2" -cat << EOF > "$OUTPUT_FILE" +cat <"$OUTPUT_FILE" providers: - name: "ck8s" url: "https://github.com/canonical/cluster-api-k8s/releases/download/${VERSION}/bootstrap-components.yaml" From 79057da3339c717b3de99016201914e51b89fb48 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Wed, 4 Sep 2024 11:40:01 -0400 Subject: [PATCH 26/79] Add Github action --- .github/workflows/e2e-deleteme.yaml | 40 ++++++++ .github/workflows/e2e.yaml | 2 + hack/b64encode_credentials.sh | 1 + hack/ci-e2e-tests.sh | 146 ++++++++++++++++++++-------- 4 files changed, 149 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/e2e-deleteme.yaml create mode 100644 hack/b64encode_credentials.sh diff --git a/.github/workflows/e2e-deleteme.yaml b/.github/workflows/e2e-deleteme.yaml new file mode 100644 index 00000000..78ea8bde --- /dev/null +++ b/.github/workflows/e2e-deleteme.yaml @@ -0,0 +1,40 @@ +name: E2E Tests + +on: + pull_request: + +permissions: + contents: read + +jobs: + run-e2e-tests: + name: Run E2E Tests + runs-on: [self-hosted, linux, X64, jammy, large] + env: + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_REGION: us-east-2 + strategy: + matrix: + ginkgo_focus: + #- "KCP remediation" + #- "MachineDeployment remediation" + - "Workload cluster creation" + #- "Workload cluster scaling" + #- "Workload cluster upgrade" + steps: + - name: Check out repo + uses: actions/checkout@v4 + - name: Install requirements + run: | + sudo apt install make + sudo apt install wget + - name: Increase inotify watches + run: | + # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files + sudo sysctl fs.inotify.max_user_watches=1048576 + sudo sysctl fs.inotify.max_user_instances=8192 + - name: Run e2e tests + run: | + sudo ./hack/ci-e2e-tests.sh true aws v0.1.2 diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2fbbd3a3..6adf9ac6 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2,6 +2,8 @@ name: E2E Tests on: pull_request: + branches: + - does-not-exist permissions: contents: read diff --git a/hack/b64encode_credentials.sh b/hack/b64encode_credentials.sh new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/hack/b64encode_credentials.sh @@ -0,0 +1 @@ + diff --git a/hack/ci-e2e-tests.sh b/hack/ci-e2e-tests.sh index 6f5b41b7..9453e4ef 100755 --- a/hack/ci-e2e-tests.sh +++ b/hack/ci-e2e-tests.sh @@ -2,20 +2,54 @@ set -xe -readonly HACK_DIR="$(realpath $(dirname "${0}"))" +# This script is used to run e2e tests for the CK8s CAPI. +# It sets up an LXD container, installs the CK8s management cluster, and runs e2e tests. +# The goal is to test the CK8s provider with different infrastructure providers (e.g., AWS, Azure, GCP). Only AWS is supported for now. +# The script should be able to run on any Linux machine with LXD installed. + +# USAGE +# ./hack/ci-e2e-tests.sh [SKIP_CLEANUP] [INFRA_PROVIDER] [CK8S_PROVIDER_VERSION] +# SKIP_CLEANUP: Optional. If set to "true", the LXD container and cloud provider resources will not be deleted after the tests are run. Default is "true". +# INFRA_PROVIDER: Optional. The infrastructure provider to use. Default is "aws". +# CK8S_PROVIDER_VERSION: Optional. The CK8s provider version to use. Default is "v0.1.2". +readonly HACK_DIR="$(realpath $(dirname "${0}"))" cd "$HACK_DIR" readonly SKIP_CLEANUP=${1:-true} -readonly +readonly INFRA_PROVIDER=${2:-aws} +readonly CK8S_PROVIDER_VERSION=${3:-v0.1.2} + readonly LXD_CHANNEL="5.21/stable" readonly LXC_IMAGE="ubuntu:20.04" readonly K8S_PROFILE_URL="https://raw.githubusercontent.com/canonical/k8s-snap/main/tests/integration/lxd-profile.yaml" readonly K8S_PROFILE_PATH="/tmp/k8s.profile" readonly CONTAINER_NAME="k8s-test" -# User specific paths -readonly USER_CREDENTIALS_PATH="/home/user/.creds" +# Utility function for printing errors to stderr +function error_exit { + printf "ERROR: %s\n" "$1" >&2 + return 1 +} + +# Check that all required environment variables are set +function check_required_env_vars { + local required_env_vars=() + + if [[ $INFRA_PROVIDER == "aws" ]]; then + required_env_vars+=("AWS_REGION" "AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY") + fi + + for var in "${required_env_vars[@]}"; do + if [ -z "${!var}" ]; then + error_exit "Missing required environment variable: $var" + fi + done +} + +function exec_in_container { + lxc exec $CONTAINER_NAME -- bash -c "$1" +} # Install LXD snap function install_lxd { @@ -35,71 +69,103 @@ function setup_lxd_profile { # Setup and configure the container function setup_container { lxc launch $LXC_IMAGE $CONTAINER_NAME -p default -p k8s - sleep 3 # Wait for the container to be up and running - lxc exec $CONTAINER_NAME -- bash -c "apt update && apt install -y snapd" - sleep 1 - lxc exec $CONTAINER_NAME -- bash -c "systemctl start snapd" + until exec_in_container true; do + sleep 1 + done + + exec_in_container "apt update && apt install -y snapd" + exec_in_container "systemctl start snapd" +} + +function configure_container_env { + if [[ $INFRA_PROVIDER == "aws" ]]; then + # Check for clusterawsadm binary + exec_in_container "which clusterawsadm" || error_exit "clusterawsadm binary not found in container" + + set +x + lxc config set $CONTAINER_NAME environment.AWS_REGION "$AWS_REGION" + lxc config set $CONTAINER_NAME environment.AWS_SECRET_ACCESS_KEY "$AWS_SECRET_ACCESS_KEY" + lxc config set $CONTAINER_NAME environment.AWS_ACCESS_KEY_ID "$AWS_ACCESS_KEY_ID" + + local aws_creds + aws_creds=$(lxc exec "$CONTAINER_NAME" -- bash -c "clusterawsadm bootstrap credentials encode-as-profile") + + lxc config set "$CONTAINER_NAME" environment.AWS_B64ENCODED_CREDENTIALS "$aws_creds" + set -x + fi } # Main installation and configuration function setup_management_cluster { - lxc exec $CONTAINER_NAME -- bash -c "snap install k8s --classic --edge" + sleep 5 + exec_in_container "snap install k8s --classic --edge" sleep 1 - lxc exec $CONTAINER_NAME -- bash -c "snap install go --classic" - lxc exec $CONTAINER_NAME -- bash -c "mkdir -p /root/.kube" - lxc exec $CONTAINER_NAME -- bash -c "sudo k8s bootstrap" - lxc exec $CONTAINER_NAME -- bash -c "sudo k8s status --wait-ready" - lxc exec $CONTAINER_NAME -- bash -c "sudo k8s config > /root/.kube/config" + exec_in_container "snap install go --classic" + exec_in_container "mkdir -p /root/.kube" + exec_in_container "sudo k8s bootstrap" + exec_in_container "sudo k8s status --wait-ready" + exec_in_container "sudo k8s config > /root/.kube/config" } # Transfer and execute scripts function install_tools { - for script in install-clusterctl.sh install-clusterctlawsadm.sh install-aws-nuke.sh write-provider-config.sh; do - lxc file push ./$script $CONTAINER_NAME/root/$script - if [[ ! $script == "write-provider-config.sh" ]]; then - lxc exec $CONTAINER_NAME -- bash -c "chmod +x /root/$script && /root/$script" - else - lxc exec $CONTAINER_NAME -- bash -c "mkdir -p /root/.cluster-api" - lxc exec $CONTAINER_NAME -- bash -c "chmod +x /root/$script && /root/$script /root/.cluster-api/clusterctl.yaml v0.1.2" - fi - done + tools=(install-clusterctl.sh) - lxc file push $USER_CREDENTIALS_PATH $CONTAINER_NAME/root/.creds - lxc exec $CONTAINER_NAME -- bash -c 'echo "source /root/.creds" >/root/.bashrc' + if [[ $INFRA_PROVIDER == "aws" ]]; then + tools+=(install-clusterctlawsadm.sh install-aws-nuke.sh) + fi + + for script in "${tools[@]}"; do + lxc file push ./"$script" $CONTAINER_NAME/root/"$script" + exec_in_container "chmod +x /root/$script && /root/$script" + done } function init_clusterctl { - lxc exec $CONTAINER_NAME -- bash -c "source /root/.creds && clusterctl init -i aws -b ck8s:v0.1.2 -c ck8s:v0.1.2 --config /root/.cluster-api/clusterctl.yaml" -} + configure_container_env # Ensures that the right environment variables are set in the container -function run_e2e_tests { - make GINKGO.FOCUS="Workload cluster creation" test-e2e + lxc file push ./write-provider-config.sh $CONTAINER_NAME/root/write-provider-config.sh + exec_in_container "chmod +x /root/write-provider-config.sh" + exec_in_container "mkdir -p /root/.cluster-api" + exec_in_container "/root/write-provider-config.sh /root/.cluster-api/clusterctl.yaml $CK8S_PROVIDER_VERSION" + + exec_in_container "clusterctl init -i $INFRA_PROVIDER -b ck8s:$CK8S_PROVIDER_VERSION -c ck8s:$CK8S_PROVIDER_VERSION --config /root/.cluster-api/clusterctl.yaml" } -function write_aws_nuke_config { - lxc exec $CONTAINER_NAME -- bash -c "mkdir -p /root/.aws-nuke" - lxc exec $CONTAINER_NAME -- bash -c "echo ""$AWS_NUKE_CONFIG"" > /root/.aws-nuke/config.yaml" +function run_e2e_tests { + make USE_EXISTING_CLUSTER=true GINKGO_FOCUS="Workload cluster creation" test-e2e } function cleanup { - write_aws_nuke_config - lxc exec $CONTAINER_NAME -- bash -c "aws-nuke --config /root/.aws-nuke/config.yaml --force" + if [[ $SKIP_CLEANUP == "true" ]]; then + return + fi + + # Infra-specific cleanup + if [[ $INFRA_PROVIDER == "aws" ]]; then + exec_in_container "mkdir -p /root/.aws-nuke" + exec_in_container "echo ""$AWS_NUKE_CONFIG"" > /root/.aws-nuke/config.yaml" + exec_in_container "aws-nuke --config /root/.aws-nuke/config.yaml --force" + fi + lxc delete $CONTAINER_NAME --force } function main { + if [[ $INFRA_PROVIDER != "aws" ]]; then + error_exit "Unsupported infrastructure provider: $INFRA_PROVIDER" + exit 1 + fi + + check_required_env_vars install_lxd setup_lxd_profile setup_container setup_management_cluster install_tools init_clusterctl - run_e2e_tests - - if [[ $SKIP_CLEANUP == "false" ]]; then - write_aws_nuke_config - cleanup - fi + #run_e2e_tests + cleanup } main From 92f4e81ad93c72728c2ca81bf56e9cd926c4389d Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 5 Sep 2024 18:35:47 -0400 Subject: [PATCH 27/79] can run tests locally in lxd container --- hack/ci-e2e-tests.sh | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/hack/ci-e2e-tests.sh b/hack/ci-e2e-tests.sh index 9453e4ef..f193697a 100755 --- a/hack/ci-e2e-tests.sh +++ b/hack/ci-e2e-tests.sh @@ -75,6 +75,9 @@ function setup_container { exec_in_container "apt update && apt install -y snapd" exec_in_container "systemctl start snapd" + + # Script is running from the hack directory, so push the entire directory to the container + lxc file push -r .. $CONTAINER_NAME/root/ } function configure_container_env { @@ -107,33 +110,46 @@ function setup_management_cluster { exec_in_container "sudo k8s config > /root/.kube/config" } +function clone_repos { + exec_in_container "git clone --depth 1 https://github.com/kubernetes-sigs/cluster-api-provider-aws /root/cluster-api-provider-aws" + exec_in_container "git clone --depth 1 https://github.com/kubernetes-sigs/cluster-api /root/cluster-api" +} + # Transfer and execute scripts function install_tools { tools=(install-clusterctl.sh) + packages=(make) + snaps=(kubectl) if [[ $INFRA_PROVIDER == "aws" ]]; then tools+=(install-clusterctlawsadm.sh install-aws-nuke.sh) fi for script in "${tools[@]}"; do - lxc file push ./"$script" $CONTAINER_NAME/root/"$script" - exec_in_container "chmod +x /root/$script && /root/$script" + exec_in_container "chmod +x /root/cluster-api-k8s/hack/$script && /root/cluster-api-k8s/hack/$script" + done + + for package in "${packages[@]}"; do + exec_in_container "apt install -y $package" + done + + for snap in "${snaps[@]}"; do + exec_in_container "snap install $snap --classic" done } function init_clusterctl { configure_container_env # Ensures that the right environment variables are set in the container - lxc file push ./write-provider-config.sh $CONTAINER_NAME/root/write-provider-config.sh - exec_in_container "chmod +x /root/write-provider-config.sh" + exec_in_container "chmod +x /root/cluster-api-k8s/hack/write-provider-config.sh" exec_in_container "mkdir -p /root/.cluster-api" - exec_in_container "/root/write-provider-config.sh /root/.cluster-api/clusterctl.yaml $CK8S_PROVIDER_VERSION" + exec_in_container "/root/cluster-api-k8s/hack/write-provider-config.sh /root/.cluster-api/clusterctl.yaml $CK8S_PROVIDER_VERSION" exec_in_container "clusterctl init -i $INFRA_PROVIDER -b ck8s:$CK8S_PROVIDER_VERSION -c ck8s:$CK8S_PROVIDER_VERSION --config /root/.cluster-api/clusterctl.yaml" } function run_e2e_tests { - make USE_EXISTING_CLUSTER=true GINKGO_FOCUS="Workload cluster creation" test-e2e + exec_in_container "cd /root/cluster-api-k8s && make USE_EXISTING_CLUSTER=true GINKGO_FOCUS=\"Workload cluster creation\" test-e2e" } function cleanup { @@ -162,9 +178,10 @@ function main { setup_lxd_profile setup_container setup_management_cluster + clone_repos install_tools init_clusterctl - #run_e2e_tests + run_e2e_tests cleanup } From f291eb8dcc76bb07c3bdd1f2fd13c028a6089c07 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 5 Sep 2024 18:42:10 -0400 Subject: [PATCH 28/79] only pass env. vars in script step --- .github/workflows/e2e-deleteme.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-deleteme.yaml b/.github/workflows/e2e-deleteme.yaml index 78ea8bde..68464b68 100644 --- a/.github/workflows/e2e-deleteme.yaml +++ b/.github/workflows/e2e-deleteme.yaml @@ -10,11 +10,6 @@ jobs: run-e2e-tests: name: Run E2E Tests runs-on: [self-hosted, linux, X64, jammy, large] - env: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - AWS_REGION: us-east-2 strategy: matrix: ginkgo_focus: @@ -37,4 +32,10 @@ jobs: sudo sysctl fs.inotify.max_user_instances=8192 - name: Run e2e tests run: | - sudo ./hack/ci-e2e-tests.sh true aws v0.1.2 + sudo -E ./hack/ci-e2e-tests.sh true aws v0.1.2 + env: + GOPROXY: direct + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_REGION: us-east-2 From fc3950b23e16c4ae492ca23b5b2b46f927d68108 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Mon, 9 Sep 2024 12:48:54 -0400 Subject: [PATCH 29/79] add debug action --- .github/workflows/e2e-deleteme.yaml | 18 +++-- hack/aws-nuke-config.yaml | 109 +++++++++++++++++++++++----- hack/ci-e2e-tests.sh | 89 +++++++++++++++-------- hack/write-provider-config.sh | 4 +- test/e2e/config/ck8s-aws.yaml | 6 +- 5 files changed, 166 insertions(+), 60 deletions(-) diff --git a/.github/workflows/e2e-deleteme.yaml b/.github/workflows/e2e-deleteme.yaml index 68464b68..984b1ad1 100644 --- a/.github/workflows/e2e-deleteme.yaml +++ b/.github/workflows/e2e-deleteme.yaml @@ -5,11 +5,12 @@ on: permissions: contents: read + id-token: write jobs: run-e2e-tests: name: Run E2E Tests - runs-on: [self-hosted, linux, X64, jammy, large] + runs-on: ubuntu-latest strategy: matrix: ginkgo_focus: @@ -19,8 +20,19 @@ jobs: #- "Workload cluster scaling" #- "Workload cluster upgrade" steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: us-east-2 + role-to-assume: arn:aws:iam::018302341396:role/GithubOIDC + role-duration-seconds: 3600 - name: Check out repo uses: actions/checkout@v4 + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + with: + detached: true - name: Install requirements run: | sudo apt install make @@ -35,7 +47,3 @@ jobs: sudo -E ./hack/ci-e2e-tests.sh true aws v0.1.2 env: GOPROXY: direct - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - AWS_REGION: us-east-2 diff --git a/hack/aws-nuke-config.yaml b/hack/aws-nuke-config.yaml index e3ec8d1e..758849b0 100644 --- a/hack/aws-nuke-config.yaml +++ b/hack/aws-nuke-config.yaml @@ -1,25 +1,96 @@ regions: - us-east-2 -account-blocklist: - - "" +blocklist: + - "999999999999" # A blocklist is required, but this is a placeholder accounts: - "": {} - + 018302341396: # AWS Account IDs are not considered sensitive information + filters: + EC2Instance: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + EC2SecurityGroup: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + EC2Volume: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + EC2InternetGateway: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + EC2NATGateway: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + EC2RouteTable: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + EC2Subnet: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + EC2VPC: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + EC2VPCEndpoint: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + EC2VPCEndpointServiceConfiguration: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + EC2ElasticIP: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + EC2NetworkInterface: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + ELBv2: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + ELBv2TargetGroup: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + S3Bucket: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + AutoScalingGroup: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true + AutoScalingLaunchConfiguration: + - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + type: "glob" + invert: true resource-types: - targets: - - EC2Instance - - EC2SecurityGroup - - EC2Volume - - EC2InternetGateway - - EC2NATGateway - - EC2RouteTable - - EC2Subnet - - EC2VPC - - EC2VPCEndpoint - - EC2VPCEndpointServiceConfiguration - - EC2ElasticIP - - EC2NetworkInterface - - ELBv2 - - ELBv2TargetGroup + includes: + - EC2Instance + - EC2SecurityGroup + - EC2Volume + - EC2InternetGateway + - EC2NATGateway + - EC2RouteTable + - EC2Subnet + - EC2VPC + - EC2VPCEndpoint + - EC2VPCEndpointServiceConfiguration + - EC2ElasticIP + - EC2NetworkInterface + - ELBv2 + - ELBv2TargetGroup + - S3Bucket + - AutoScalingGroup + - AutoScalingLaunchConfiguration diff --git a/hack/ci-e2e-tests.sh b/hack/ci-e2e-tests.sh index f193697a..f3b753e1 100755 --- a/hack/ci-e2e-tests.sh +++ b/hack/ci-e2e-tests.sh @@ -1,6 +1,8 @@ #!/bin/bash -set -xe +# WARNING: DO NOT enable -x as it will expose sensitive information in the logs. +# Enable debugging selectively using set -x and set +x around specific code blocks. +set -euo pipefail # This script is used to run e2e tests for the CK8s CAPI. # It sets up an LXD container, installs the CK8s management cluster, and runs e2e tests. @@ -8,20 +10,18 @@ set -xe # The script should be able to run on any Linux machine with LXD installed. # USAGE -# ./hack/ci-e2e-tests.sh [SKIP_CLEANUP] [INFRA_PROVIDER] [CK8S_PROVIDER_VERSION] -# SKIP_CLEANUP: Optional. If set to "true", the LXD container and cloud provider resources will not be deleted after the tests are run. Default is "true". +# ./hack/ci-e2e-tests.sh [INFRA_PROVIDER] [CK8S_PROVIDER_VERSION] # INFRA_PROVIDER: Optional. The infrastructure provider to use. Default is "aws". # CK8S_PROVIDER_VERSION: Optional. The CK8s provider version to use. Default is "v0.1.2". readonly HACK_DIR="$(realpath $(dirname "${0}"))" cd "$HACK_DIR" -readonly SKIP_CLEANUP=${1:-true} readonly INFRA_PROVIDER=${2:-aws} readonly CK8S_PROVIDER_VERSION=${3:-v0.1.2} -readonly LXD_CHANNEL="5.21/stable" -readonly LXC_IMAGE="ubuntu:20.04" +readonly LXD_CHANNEL="6.1/stable" +readonly LXC_IMAGE="ubuntu:22.04" readonly K8S_PROFILE_URL="https://raw.githubusercontent.com/canonical/k8s-snap/main/tests/integration/lxd-profile.yaml" readonly K8S_PROFILE_PATH="/tmp/k8s.profile" readonly CONTAINER_NAME="k8s-test" @@ -32,8 +32,14 @@ function error_exit { return 1 } +function log_info { + printf "INFO: %s\n" "$1" +} + # Check that all required environment variables are set function check_required_env_vars { + log_info "Checking required environment variables..." + local required_env_vars=() if [[ $INFRA_PROVIDER == "aws" ]]; then @@ -51,16 +57,30 @@ function exec_in_container { lxc exec $CONTAINER_NAME -- bash -c "$1" } +function setup_firewall { + log_info "Setting up firewall rules..." + + if sudo iptables -L DOCKER-USER; then + sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT + sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + fi +} + # Install LXD snap function install_lxd { + log_info "Installing LXD..." + sudo snap install lxd --channel=$LXD_CHANNEL + sudo lxd waitready sudo lxd init --auto sudo usermod --append --groups lxd "$USER" } # Create or ensure the k8s profile exists function setup_lxd_profile { - lxc profile create k8s || true + log_info "Setting up LXD profile..." + + lxc profile show k8s || lxc profile create k8s wget -q $K8S_PROFILE_URL -O $K8S_PROFILE_PATH cat $K8S_PROFILE_PATH | lxc profile edit k8s rm -f $K8S_PROFILE_PATH @@ -68,42 +88,58 @@ function setup_lxd_profile { # Setup and configure the container function setup_container { + log_info "Setting up LXD container..." + lxc launch $LXC_IMAGE $CONTAINER_NAME -p default -p k8s + + # Wait for container to be ready to run commands until exec_in_container true; do sleep 1 done exec_in_container "apt update && apt install -y snapd" exec_in_container "systemctl start snapd" + exec_in_container "snap wait core seed.loaded" # Script is running from the hack directory, so push the entire directory to the container - lxc file push -r .. $CONTAINER_NAME/root/ + lxc file push -r .. $CONTAINER_NAME/root/ >/dev/null } function configure_container_env { + log_info "Configuring container environment..." + if [[ $INFRA_PROVIDER == "aws" ]]; then + log_info "Configuring AWS credentials in container..." + # Check for clusterawsadm binary exec_in_container "which clusterawsadm" || error_exit "clusterawsadm binary not found in container" - set +x lxc config set $CONTAINER_NAME environment.AWS_REGION "$AWS_REGION" lxc config set $CONTAINER_NAME environment.AWS_SECRET_ACCESS_KEY "$AWS_SECRET_ACCESS_KEY" lxc config set $CONTAINER_NAME environment.AWS_ACCESS_KEY_ID "$AWS_ACCESS_KEY_ID" + if [[ -z $AWS_SESSION_TOKEN ]]; then + log_info "AWS_SESSION_TOKEN not set. Skipping..." + else + lxc config set $CONTAINER_NAME environment.AWS_SESSION_TOKEN "$AWS_SESSION_TOKEN" + fi + + # This command can fail if the stack already exists, so we ignore the error + exec_in_container "clusterawsadm bootstrap iam create-cloudformation-stack" || true + local aws_creds aws_creds=$(lxc exec "$CONTAINER_NAME" -- bash -c "clusterawsadm bootstrap credentials encode-as-profile") + echo "::add-mask::$aws_creds" # Mask the credentials in the Github CI logs. lxc config set "$CONTAINER_NAME" environment.AWS_B64ENCODED_CREDENTIALS "$aws_creds" - set -x fi } # Main installation and configuration function setup_management_cluster { - sleep 5 - exec_in_container "snap install k8s --classic --edge" - sleep 1 - exec_in_container "snap install go --classic" + log_info "Setting up management cluster..." + exec_in_container "sudo snap install k8s --classic --edge" + exec_in_container "sudo snap install go --classic" exec_in_container "mkdir -p /root/.kube" exec_in_container "sudo k8s bootstrap" exec_in_container "sudo k8s status --wait-ready" @@ -111,12 +147,15 @@ function setup_management_cluster { } function clone_repos { + log_info "Cloning CK8s and CAPI repositories..." exec_in_container "git clone --depth 1 https://github.com/kubernetes-sigs/cluster-api-provider-aws /root/cluster-api-provider-aws" exec_in_container "git clone --depth 1 https://github.com/kubernetes-sigs/cluster-api /root/cluster-api" } # Transfer and execute scripts function install_tools { + log_info "Installing tools in container..." + tools=(install-clusterctl.sh) packages=(make) snaps=(kubectl) @@ -139,6 +178,8 @@ function install_tools { } function init_clusterctl { + log_info "Initializing clusterctl with $INFRA_PROVIDER infrastructure and CK8s $CK8S_PROVIDER_VERSION..." + configure_container_env # Ensures that the right environment variables are set in the container exec_in_container "chmod +x /root/cluster-api-k8s/hack/write-provider-config.sh" @@ -149,24 +190,10 @@ function init_clusterctl { } function run_e2e_tests { + log_info "Running e2e tests..." exec_in_container "cd /root/cluster-api-k8s && make USE_EXISTING_CLUSTER=true GINKGO_FOCUS=\"Workload cluster creation\" test-e2e" } -function cleanup { - if [[ $SKIP_CLEANUP == "true" ]]; then - return - fi - - # Infra-specific cleanup - if [[ $INFRA_PROVIDER == "aws" ]]; then - exec_in_container "mkdir -p /root/.aws-nuke" - exec_in_container "echo ""$AWS_NUKE_CONFIG"" > /root/.aws-nuke/config.yaml" - exec_in_container "aws-nuke --config /root/.aws-nuke/config.yaml --force" - fi - - lxc delete $CONTAINER_NAME --force -} - function main { if [[ $INFRA_PROVIDER != "aws" ]]; then error_exit "Unsupported infrastructure provider: $INFRA_PROVIDER" @@ -176,13 +203,15 @@ function main { check_required_env_vars install_lxd setup_lxd_profile + setup_firewall setup_container setup_management_cluster clone_repos install_tools init_clusterctl run_e2e_tests - cleanup + + log_info "E2E tests completed successfully." } main diff --git a/hack/write-provider-config.sh b/hack/write-provider-config.sh index c396216b..38bab5c7 100644 --- a/hack/write-provider-config.sh +++ b/hack/write-provider-config.sh @@ -6,9 +6,7 @@ # Usage: # $ write-clusterctl-config.sh $output-file $version -set -o errexit -set -o nounset -set -o pipefail +set -euo pipefail OUTPUT_FILE="$1" VERSION="$2" diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index c80c540b..1f93e009 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -32,7 +32,7 @@ providers: # By default, will use the latest version defined in ../data/shared/v1beta1/metadata.yaml # to init the management cluster - name: v2.6.1 # used during e2e-test - value: "../../../../cluster-api-provider-aws/config/default" + value: "../../../../cluster-api-provider-aws/config/default" # TODO don't use relative path contract: v1beta2 files: - sourcePath: "../data/shared/v1beta1_aws/metadata.yaml" @@ -46,7 +46,7 @@ providers: # default version for docker infrastructure provider # name here should match defaultProviderVersion - name: v1.9.99 - value: "../../../../cluster-api-provider-aws/config/default" + value: "../../../../cluster-api-provider-aws/config/default" # TODO don't use relative path contract: v1beta2 files: - sourcePath: "../data/shared/v1beta1_aws/metadata.yaml" @@ -95,7 +95,7 @@ variables: AWS_NODE_INSTANCE_TYPE: t3.large AWS_PUBLIC_IP: false AWS_CREATE_BASTION: true - AWS_SSH_KEY_NAME: "etienne" + AWS_SSH_KEY_NAME: "" AWS_AMI_ID: "ami-05145146e3a9db6f3" AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE: 16 AWS_NODE_ROOT_VOLUME_SIZE: 16 From 250584e8d9b4d9c7aa3b41a45dde7794328eea6f Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Wed, 11 Sep 2024 22:42:01 -0400 Subject: [PATCH 30/79] delete file --- hack/b64encode_credentials.sh | 1 - 1 file changed, 1 deletion(-) delete mode 100644 hack/b64encode_credentials.sh diff --git a/hack/b64encode_credentials.sh b/hack/b64encode_credentials.sh deleted file mode 100644 index 8b137891..00000000 --- a/hack/b64encode_credentials.sh +++ /dev/null @@ -1 +0,0 @@ - From 786b3b4c558f291a5a93e3c2d6e33314b0643fdb Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Wed, 11 Sep 2024 22:48:00 -0400 Subject: [PATCH 31/79] update comments --- test/e2e/config/ck8s-aws.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index 1f93e009..dfaa03b4 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -32,7 +32,7 @@ providers: # By default, will use the latest version defined in ../data/shared/v1beta1/metadata.yaml # to init the management cluster - name: v2.6.1 # used during e2e-test - value: "../../../../cluster-api-provider-aws/config/default" # TODO don't use relative path + value: "../../../../cluster-api-provider-aws/config/default" # TODO don't use relative path? contract: v1beta2 files: - sourcePath: "../data/shared/v1beta1_aws/metadata.yaml" @@ -85,6 +85,7 @@ providers: - sourcePath: "../../../metadata.yaml" targetName: "metadata.yaml" +# These variables replace the variables in test/e2e/data/infrastructure-aws manifests variables: KUBERNETES_VERSION_MANAGEMENT: "v1.30.0" KUBERNETES_VERSION: "v1.30.0" @@ -102,6 +103,8 @@ variables: AWS_REGION: "us-east-2" AWS_CCM_IMAGE: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.30.0" # https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml#L203C1-L205C27 + # There is some work to be done here on figuring out which experimental features + # we want to enable/disable. EXP_CLUSTER_RESOURCE_SET: "true" #EXP_RUNTIME_SDK: "true" EXP_MACHINE_SET_PREFLIGHT_CHECKS: "false" @@ -109,7 +112,6 @@ variables: #CLUSTER_TOPOLOGY: "true" CAPA_LOGLEVEL: "4" - intervals: # copied from https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml default/wait-cluster: [ "35m", "10s" ] From cfc5949eded99bd52f25a6f17d95585755cdd355 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Wed, 25 Sep 2024 16:13:07 -0400 Subject: [PATCH 32/79] test --- .github/workflows/e2e-deleteme.yaml | 15 ++++++++------- .github/workflows/e2e.yaml | 2 -- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e-deleteme.yaml b/.github/workflows/e2e-deleteme.yaml index 984b1ad1..9af1ca9a 100644 --- a/.github/workflows/e2e-deleteme.yaml +++ b/.github/workflows/e2e-deleteme.yaml @@ -2,6 +2,8 @@ name: E2E Tests on: pull_request: + branches: + - does-not-exist permissions: contents: read @@ -14,11 +16,11 @@ jobs: strategy: matrix: ginkgo_focus: - #- "KCP remediation" - #- "MachineDeployment remediation" + - "KCP remediation" + - "MachineDeployment remediation" - "Workload cluster creation" - #- "Workload cluster scaling" - #- "Workload cluster upgrade" + - "Workload cluster scaling" + - "Workload cluster upgrade" steps: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -35,8 +37,7 @@ jobs: detached: true - name: Install requirements run: | - sudo apt install make - sudo apt install wget + sudo apt install make wget - name: Increase inotify watches run: | # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files @@ -44,6 +45,6 @@ jobs: sudo sysctl fs.inotify.max_user_instances=8192 - name: Run e2e tests run: | - sudo -E ./hack/ci-e2e-tests.sh true aws v0.1.2 + sudo -E ./hack/ci-e2e-tests.sh aws v0.1.2 env: GOPROXY: direct diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 6adf9ac6..2fbbd3a3 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2,8 +2,6 @@ name: E2E Tests on: pull_request: - branches: - - does-not-exist permissions: contents: read From 4746fd5e61e39fd8b9971914864e8e12f124d67c Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 26 Sep 2024 15:37:06 -0400 Subject: [PATCH 33/79] cleanup ck8s-aws.yaml --- test/e2e/config/ck8s-aws.yaml | 33 ++++++++++------------ test/e2e/data/shared/v1beta1/metadata.yaml | 3 ++ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index dfaa03b4..b27a25d1 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -2,10 +2,8 @@ managementClusterName: capi-test # E2E test scenario using local dev images and manifests built from the source tree for following providers: -# - cluster-api # - bootstrap ck8s # - control-plane ck8s -# - aws images: # Use local dev images built source tree; - name: ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev @@ -13,12 +11,14 @@ images: - name: ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev loadBehavior: mustLoad +# List of providers that will be installed into the management cluster +# See InitManagementClusterAndWatchControllerLogs function call providers: - name: cluster-api type: CoreProvider versions: - - name: v1.7.1 - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.1/core-components.yaml + - name: v1.8.3 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.3/core-components.yaml type: url contract: v1beta1 files: @@ -29,10 +29,9 @@ providers: - name: aws type: InfrastructureProvider versions: - # By default, will use the latest version defined in ../data/shared/v1beta1/metadata.yaml - # to init the management cluster - - name: v2.6.1 # used during e2e-test - value: "../../../../cluster-api-provider-aws/config/default" # TODO don't use relative path? + - name: v2.6.1 + value: "https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.6.1/infrastructure-components.yaml" + type: url contract: v1beta2 files: - sourcePath: "../data/shared/v1beta1_aws/metadata.yaml" @@ -40,13 +39,12 @@ providers: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" - # Add v1.9.99 to support tilt (not presented in ../data/shared/v1beta1/metadata.yaml) # when bootstrapping with tilt, it will use - # the defaultProviderVersion in https://github.com/kubernetes-sigs/cluster-api/blob/main/hack/tools/internal/tilt-prepare/main.go as - # default version for docker infrastructure provider + # https://github.com/kubernetes-sigs/cluster-api/blob/main/hack/tools/internal/tilt-prepare/main.go # name here should match defaultProviderVersion - name: v1.9.99 - value: "../../../../cluster-api-provider-aws/config/default" # TODO don't use relative path + value: "https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.6.1/infrastructure-components.yaml" + type: url contract: v1beta2 files: - sourcePath: "../data/shared/v1beta1_aws/metadata.yaml" @@ -69,10 +67,9 @@ providers: value: "../../../bootstrap/config/default" replacements: - old: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:latest" - new: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev" + new: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:v0.1.2-amd64" files: - sourcePath: "../../../metadata.yaml" - targetName: "metadata.yaml" - name: ck8s type: ControlPlaneProvider versions: @@ -80,12 +77,12 @@ providers: value: "../../../controlplane/config/default" replacements: - old: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:latest" - new: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev" + new: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:v0.1.2-amd64" files: - sourcePath: "../../../metadata.yaml" - targetName: "metadata.yaml" # These variables replace the variables in test/e2e/data/infrastructure-aws manifests +# They are used during clusterctl generate cluster variables: KUBERNETES_VERSION_MANAGEMENT: "v1.30.0" KUBERNETES_VERSION: "v1.30.0" @@ -96,7 +93,7 @@ variables: AWS_NODE_INSTANCE_TYPE: t3.large AWS_PUBLIC_IP: false AWS_CREATE_BASTION: true - AWS_SSH_KEY_NAME: "" + AWS_SSH_KEY_NAME: "etienne" AWS_AMI_ID: "ami-05145146e3a9db6f3" AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE: 16 AWS_NODE_ROOT_VOLUME_SIZE: 16 @@ -113,7 +110,7 @@ variables: CAPA_LOGLEVEL: "4" intervals: - # copied from https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml + # Ref: https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml default/wait-cluster: [ "35m", "10s" ] default/wait-control-plane: [ "35m", "10s" ] default/wait-worker-nodes: [ "20m", "10s" ] diff --git a/test/e2e/data/shared/v1beta1/metadata.yaml b/test/e2e/data/shared/v1beta1/metadata.yaml index c1a68366..b7d38077 100644 --- a/test/e2e/data/shared/v1beta1/metadata.yaml +++ b/test/e2e/data/shared/v1beta1/metadata.yaml @@ -1,6 +1,9 @@ apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 kind: Metadata releaseSeries: + - major: 1 + minor: 8 + contract: v1beta1 - major: 1 minor: 7 contract: v1beta1 From 081b14a5e3e1579f552b8cac9697396046dd33fd Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 26 Sep 2024 16:28:23 -0400 Subject: [PATCH 34/79] don't use custom script --- .github/workflows/e2e-deleteme.yaml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e-deleteme.yaml b/.github/workflows/e2e-deleteme.yaml index 9af1ca9a..08d4c8fa 100644 --- a/.github/workflows/e2e-deleteme.yaml +++ b/.github/workflows/e2e-deleteme.yaml @@ -13,15 +13,15 @@ jobs: run-e2e-tests: name: Run E2E Tests runs-on: ubuntu-latest - strategy: - matrix: - ginkgo_focus: - - "KCP remediation" - - "MachineDeployment remediation" - - "Workload cluster creation" - - "Workload cluster scaling" - - "Workload cluster upgrade" steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ~1.22.0 + - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -29,22 +29,27 @@ jobs: aws-region: us-east-2 role-to-assume: arn:aws:iam::018302341396:role/GithubOIDC role-duration-seconds: 3600 + - name: Check out repo uses: actions/checkout@v4 + - name: Setup tmate session uses: mxschmitt/action-tmate@v3 with: detached: true + - name: Install requirements run: | sudo apt install make wget + - name: Increase inotify watches run: | # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files sudo sysctl fs.inotify.max_user_watches=1048576 sudo sysctl fs.inotify.max_user_instances=8192 + - name: Run e2e tests run: | - sudo -E ./hack/ci-e2e-tests.sh aws v0.1.2 + GINKGO_FOCUS="Workload cluster creation" make test-e2e env: GOPROXY: direct From 718caa7d609840d9251baf5521b9f21d013f0692 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 26 Sep 2024 16:36:00 -0400 Subject: [PATCH 35/79] launch test on aws --- .github/workflows/e2e-deleteme.yaml | 14 +++++++++++--- .github/workflows/e2e.yaml | 2 ++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-deleteme.yaml b/.github/workflows/e2e-deleteme.yaml index 08d4c8fa..a8f9d24a 100644 --- a/.github/workflows/e2e-deleteme.yaml +++ b/.github/workflows/e2e-deleteme.yaml @@ -2,8 +2,6 @@ name: E2E Tests on: pull_request: - branches: - - does-not-exist permissions: contents: read @@ -41,6 +39,9 @@ jobs: - name: Install requirements run: | sudo apt install make wget + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind - name: Increase inotify watches run: | @@ -48,8 +49,15 @@ jobs: sudo sysctl fs.inotify.max_user_watches=1048576 sudo sysctl fs.inotify.max_user_instances=8192 + - name: Build images + run: | + cd hack + ./build-e2e-images.sh + cd .. + make docker-build-e2e + - name: Run e2e tests run: | - GINKGO_FOCUS="Workload cluster creation" make test-e2e + E2E_INFRA=aws GINKGO_FOCUS="Workload cluster creation" make test-e2e env: GOPROXY: direct diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2fbbd3a3..6adf9ac6 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2,6 +2,8 @@ name: E2E Tests on: pull_request: + branches: + - does-not-exist permissions: contents: read From fa793f88b53f0de41cd737c2c3b4ebb8affe20c6 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 15 Oct 2024 18:40:59 -0400 Subject: [PATCH 36/79] tidy --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index c52d39e8..4e11c408 100644 --- a/go.mod +++ b/go.mod @@ -91,6 +91,7 @@ require ( github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect From 93a6607eff172b0920798721de76700965ed5955 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Wed, 16 Oct 2024 15:47:00 -0400 Subject: [PATCH 37/79] update templates --- .../cluster-template-kcp-remediation.yaml | 41 ++++--------------- .../cluster-template-md-remediation.yaml | 28 +------------ .../infrastructure-aws/cluster-template.yaml | 32 ++------------- 3 files changed, 14 insertions(+), 87 deletions(-) diff --git a/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml index 3862314e..6c463455 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml @@ -52,41 +52,28 @@ spec: name: ${CLUSTER_NAME}-control-plane spec: nodeName: "{{ ds.meta_data.local_hostname }}" - preRunCommands: - - systemctl stop kubelet || true - - systemctl disable kubelet || true - - systemctl stop containerd || true - - systemctl disable containerd || true - - ./wait-signal.sh "${TOKEN}" "${SERVER}" "${NAMESPACE}" + channel: "1.31-classic/edge" files: - # note(ben): This is only required as long as k8s does not have a stable release. - - path: /capi/scripts/install.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - snap install k8s --classic --channel=latest/edge/etienne-delete-me - - path: /wait-signal.sh content: | #!/bin/bash - + set -o errexit set -o pipefail - + echo "Waiting for signal..." - + TOKEN=$1 SERVER=$2 NAMESPACE=$3 - + while true; do sleep 1s - + signal=$(curl -k -s --header "Authorization: Bearer $TOKEN" $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test | jq -r .data.signal?) echo "signal $signal" - + if [ "$signal" == "pass" ]; then curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test exit 0 @@ -163,19 +150,7 @@ spec: template: spec: nodeName: "{{ ds.meta_data.local_hostname }}" - preRunCommands: - - systemctl stop kubelet || true - - systemctl disable kubelet || true - - systemctl stop containerd || true - - systemctl disable containerd || true - files: - # note(ben): This is only required as long as k8s does not have a stable release. - - path: /capi/scripts/install.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - snap install k8s --classic --channel=latest/edge/etienne-delete-me + channel: "1.31-classic/edge" --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet diff --git a/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml index c2211828..05b93a02 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml @@ -52,19 +52,7 @@ spec: name: ${CLUSTER_NAME}-control-plane spec: nodeName: "{{ ds.meta_data.local_hostname }}" - preRunCommands: - - systemctl stop kubelet || true - - systemctl disable kubelet || true - - systemctl stop containerd || true - - systemctl disable containerd || true - files: - # note(ben): This is only required as long as k8s does not have a stable release. - - path: /capi/scripts/install.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - snap install k8s --classic --channel=latest/edge/etienne-delete-me + channel: "1.31-classic/edge" controlPlane: cloudProvider: external replicas: ${CONTROL_PLANE_MACHINE_COUNT} @@ -139,19 +127,7 @@ spec: template: spec: nodeName: "{{ ds.meta_data.local_hostname }}" - preRunCommands: - - systemctl stop kubelet || true - - systemctl disable kubelet || true - - systemctl stop containerd || true - - systemctl disable containerd || true - files: - # note(ben): This is only required as long as k8s does not have a stable release. - - path: /capi/scripts/install.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - snap install k8s --classic --channel=latest/edge/etienne-delete-me + channel: "1.31-classic/edge" --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet diff --git a/test/e2e/data/infrastructure-aws/cluster-template.yaml b/test/e2e/data/infrastructure-aws/cluster-template.yaml index 057ece5c..fc5b8561 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template.yaml @@ -8,10 +8,10 @@ spec: clusterNetwork: pods: cidrBlocks: - - 10.1.0.0/16 + - 10.1.0.0/16 services: cidrBlocks: - - 10.152.183.0/24 + - 10.152.183.0/24 controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane @@ -52,19 +52,7 @@ spec: name: ${CLUSTER_NAME}-control-plane spec: nodeName: "{{ ds.meta_data.local_hostname }}" - preRunCommands: - - systemctl stop kubelet || true - - systemctl disable kubelet || true - - systemctl stop containerd || true - - systemctl disable containerd || true - files: - # note(ben): This is only required as long as k8s does not have a stable release. - - path: /capi/scripts/install.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - snap install k8s --classic --channel=latest/edge/etienne-delete-me + channel: "1.31-classic/edge" controlPlane: cloudProvider: external replicas: ${CONTROL_PLANE_MACHINE_COUNT} @@ -134,19 +122,7 @@ spec: template: spec: nodeName: "{{ ds.meta_data.local_hostname }}" - preRunCommands: - - systemctl stop kubelet || true - - systemctl disable kubelet || true - - systemctl stop containerd || true - - systemctl disable containerd || true - files: - # note(ben): This is only required as long as k8s does not have a stable release. - - path: /capi/scripts/install.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - snap install k8s --classic --channel=latest/edge/etienne-delete-me + channel: "1.31-classic/edge" --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet From 4919b04f8aec927ea7d2ed5e8c6f3119a65af8cb Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 17 Oct 2024 13:13:19 -0400 Subject: [PATCH 38/79] use dev tag so locally built images are used --- .github/workflows/e2e-deleteme.yaml | 2 +- test/e2e/config/ck8s-aws.yaml | 16 ++++++++-------- .../cluster-template-kcp-remediation.yaml | 1 + .../cluster-template-md-remediation.yaml | 1 + 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e-deleteme.yaml b/.github/workflows/e2e-deleteme.yaml index a8f9d24a..0df06e6f 100644 --- a/.github/workflows/e2e-deleteme.yaml +++ b/.github/workflows/e2e-deleteme.yaml @@ -58,6 +58,6 @@ jobs: - name: Run e2e tests run: | - E2E_INFRA=aws GINKGO_FOCUS="Workload cluster creation" make test-e2e + E2E_INFRA=aws GINKGO_FOCUS="KCP remediation" make test-e2e env: GOPROXY: direct diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index b27a25d1..64201aa3 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -67,7 +67,7 @@ providers: value: "../../../bootstrap/config/default" replacements: - old: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:latest" - new: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:v0.1.2-amd64" + new: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev" files: - sourcePath: "../../../metadata.yaml" - name: ck8s @@ -77,7 +77,7 @@ providers: value: "../../../controlplane/config/default" replacements: - old: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:latest" - new: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:v0.1.2-amd64" + new: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev" files: - sourcePath: "../../../metadata.yaml" @@ -92,9 +92,9 @@ variables: AWS_CONTROL_PLANE_INSTANCE_TYPE: t3.large AWS_NODE_INSTANCE_TYPE: t3.large AWS_PUBLIC_IP: false - AWS_CREATE_BASTION: true - AWS_SSH_KEY_NAME: "etienne" - AWS_AMI_ID: "ami-05145146e3a9db6f3" + AWS_CREATE_BASTION: false + AWS_SSH_KEY_NAME: "default" + AWS_AMI_ID: "ami-01b139e6226d65e4f" AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE: 16 AWS_NODE_ROOT_VOLUME_SIZE: 16 AWS_REGION: "us-east-2" @@ -103,10 +103,10 @@ variables: # There is some work to be done here on figuring out which experimental features # we want to enable/disable. EXP_CLUSTER_RESOURCE_SET: "true" - #EXP_RUNTIME_SDK: "true" + EXP_RUNTIME_SDK: "true" EXP_MACHINE_SET_PREFLIGHT_CHECKS: "false" - #EXP_MACHINE_POOL: "true" - #CLUSTER_TOPOLOGY: "true" + EXP_MACHINE_POOL: "true" + CLUSTER_TOPOLOGY: "true" CAPA_LOGLEVEL: "4" intervals: diff --git a/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml index 6c463455..0bd70c75 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml @@ -31,6 +31,7 @@ spec: bastion: enabled: ${AWS_CREATE_BASTION} controlPlaneLoadBalancer: + loadBalancerType: nlb healthCheckProtocol: TCP network: cni: diff --git a/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml index 05b93a02..4177c710 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml @@ -31,6 +31,7 @@ spec: bastion: enabled: ${AWS_CREATE_BASTION} controlPlaneLoadBalancer: + loadBalancerType: nlb healthCheckProtocol: TCP network: cni: From 06fbc70d078158bd108754714e7a7f315e6f80f6 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 17 Oct 2024 13:39:01 -0400 Subject: [PATCH 39/79] use self-hosted --- .github/workflows/e2e-deleteme.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-deleteme.yaml b/.github/workflows/e2e-deleteme.yaml index 0df06e6f..b6c31c46 100644 --- a/.github/workflows/e2e-deleteme.yaml +++ b/.github/workflows/e2e-deleteme.yaml @@ -9,8 +9,8 @@ permissions: jobs: run-e2e-tests: + runs-on: [self-hosted, linux, X64, jammy, large] name: Run E2E Tests - runs-on: ubuntu-latest steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -31,10 +31,18 @@ jobs: - name: Check out repo uses: actions/checkout@v4 - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 + # with: + # detached: true + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 with: - detached: true + # We run into rate limiting issues if we don't authenticate + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Install requirements run: | From fcef40f0ea53e668a7088ab284de6b3ba0eea9a7 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 17 Oct 2024 16:41:07 -0400 Subject: [PATCH 40/79] skip remediation tests on aws, add aws infra in action --- .github/workflows/e2e.yaml | 23 +- test/e2e/config/ck8s-aws.yaml | 3 +- .../cluster-template-kcp-remediation.yaml | 363 ------------------ .../cluster-template-md-remediation.yaml | 340 ---------------- test/e2e/kcp_remediation_test.go | 10 +- test/e2e/md_remediation_test.go | 7 + 6 files changed, 33 insertions(+), 713 deletions(-) delete mode 100644 test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml delete mode 100644 test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 6adf9ac6..c9315b1a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -13,8 +13,7 @@ jobs: name: Build & Run E2E Images runs-on: [self-hosted, linux, X64, jammy, large] steps: - - - name: Login to GitHub Container Registry + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: # We run into rate limiting issues if we don't authenticate @@ -26,9 +25,8 @@ jobs: - name: Install requirements run: | sudo apt update + sudo apt install make docker-buildx sudo snap install go --classic --channel=1.22/stable - sudo apt install make - sudo apt install docker-buildx sudo snap install kubectl --classic --channel=1.30/stable - name: Build provider images run: sudo make docker-build-e2e @@ -61,6 +59,9 @@ jobs: needs: build-e2e-images strategy: matrix: + infra: + - "aws" + #- "docker" ginkgo_focus: - "KCP remediation" - "MachineDeployment remediation" @@ -70,8 +71,7 @@ jobs: # TODO(ben): Remove once all tests are running stable. fail-fast: false steps: - - - name: Login to GitHub Container Registry + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: # We run into rate limiting issues if we don't authenticate @@ -109,6 +109,15 @@ jobs: # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files sudo sysctl fs.inotify.max_user_watches=1048576 sudo sysctl fs.inotify.max_user_instances=8192 + - name: Configure AWS Credentials + if: matrix.infra == 'aws' + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: us-east-2 + role-to-assume: arn:aws:iam::018302341396:role/GithubOIDC + role-duration-seconds: 3600 - name: Run e2e tests + if: ${{!(matrix.infra == 'aws' && (matrix.ginkgo_focus == 'KCP remediation' || matrix.ginkgo_focus == 'MachineDeployment remediation'))}} run: | - sudo GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e + sudo E2E_INFRA=${{matrix.infra}} GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index 64201aa3..ee61c704 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -53,8 +53,6 @@ providers: new: "imagePullPolicy: IfNotPresent" files: - sourcePath: "../data/infrastructure-aws/cluster-template.yaml" - - sourcePath: "../data/infrastructure-aws/cluster-template-md-remediation.yaml" - - sourcePath: "../data/infrastructure-aws/cluster-template-kcp-remediation.yaml" - name: ck8s type: BootstrapProvider versions: @@ -111,6 +109,7 @@ variables: intervals: # Ref: https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml + default/wait-machines: [ "35m", "10s" ] default/wait-cluster: [ "35m", "10s" ] default/wait-control-plane: [ "35m", "10s" ] default/wait-worker-nodes: [ "20m", "10s" ] diff --git a/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml deleted file mode 100644 index 0bd70c75..00000000 --- a/test/e2e/data/infrastructure-aws/cluster-template-kcp-remediation.yaml +++ /dev/null @@ -1,363 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - labels: - ccm: external -spec: - clusterNetwork: - pods: - cidrBlocks: - - 10.1.0.0/16 - services: - cidrBlocks: - - 10.152.183.0/24 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta2 - kind: CK8sControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: AWSCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: AWSCluster -metadata: - name: ${CLUSTER_NAME} -spec: - region: ${AWS_REGION} - sshKeyName: ${AWS_SSH_KEY_NAME} - bastion: - enabled: ${AWS_CREATE_BASTION} - controlPlaneLoadBalancer: - loadBalancerType: nlb - healthCheckProtocol: TCP - network: - cni: - cniIngressRules: - - description: microcluster - protocol: tcp - toPort: 2380 ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta2 -kind: CK8sControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - machineTemplate: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: AWSMachineTemplate - name: ${CLUSTER_NAME}-control-plane - spec: - nodeName: "{{ ds.meta_data.local_hostname }}" - channel: "1.31-classic/edge" - files: - - path: /wait-signal.sh - content: | - #!/bin/bash - - set -o errexit - set -o pipefail - - echo "Waiting for signal..." - - TOKEN=$1 - SERVER=$2 - NAMESPACE=$3 - - while true; - do - sleep 1s - - signal=$(curl -k -s --header "Authorization: Bearer $TOKEN" $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test | jq -r .data.signal?) - echo "signal $signal" - - if [ "$signal" == "pass" ]; then - curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test - exit 0 - fi - done - permissions: "0777" - owner: root:root - controlPlane: - cloudProvider: external - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: AWSMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane -spec: - template: - spec: - ami: - id: ${AWS_AMI_ID} - iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io - instanceType: ${AWS_CONTROL_PLANE_INSTANCE_TYPE} - publicIP: ${AWS_PUBLIC_IP} - sshKeyName: ${AWS_SSH_KEY_NAME} - rootVolume: - size: ${AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-worker-md-0 -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} - template: - spec: - version: ${KUBERNETES_VERSION} - clusterName: ${CLUSTER_NAME} - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 - kind: CK8sConfigTemplate - name: ${CLUSTER_NAME}-md-0 - infrastructureRef: - name: "${CLUSTER_NAME}-md-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: AWSMachineTemplate ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: AWSMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 -spec: - template: - spec: - ami: - id: ${AWS_AMI_ID} - iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io - instanceType: ${AWS_NODE_INSTANCE_TYPE} - publicIP: ${AWS_PUBLIC_IP} - sshKeyName: ${AWS_SSH_KEY_NAME} - rootVolume: - size: ${AWS_NODE_ROOT_VOLUME_SIZE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 -kind: CK8sConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 -spec: - template: - spec: - nodeName: "{{ ds.meta_data.local_hostname }}" - channel: "1.31-classic/edge" ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: crs-ccm -spec: - clusterSelector: - matchLabels: - ccm: external - resources: - - kind: ConfigMap - name: cloud-controller-manager-addon - strategy: ApplyOnce ---- -apiVersion: v1 -data: - aws-ccm-external.yaml: | - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: aws-cloud-controller-manager - namespace: kube-system - labels: - k8s-app: aws-cloud-controller-manager - spec: - selector: - matchLabels: - k8s-app: aws-cloud-controller-manager - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - k8s-app: aws-cloud-controller-manager - spec: - nodeSelector: - node-role.kubernetes.io/control-plane: "" - tolerations: - - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - effect: NoSchedule - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - serviceAccountName: cloud-controller-manager - containers: - - name: aws-cloud-controller-manager - image: ${AWS_CCM_IMAGE} - args: - - --v=2 - - --cloud-provider=aws - - --use-service-account-credentials=true - - --configure-cloud-routes=false - resources: - requests: - cpu: 200m - hostNetwork: true - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: cloud-controller-manager - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: cloud-controller-manager:apiserver-authentication-reader - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader - subjects: - - apiGroup: "" - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: system:cloud-controller-manager - rules: - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - '*' - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - services - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - services/status - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - get - - list - - watch - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - update - - watch - - apiGroups: - - "" - resources: - - endpoints - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - serviceaccounts/token - verbs: - - create - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: system:cloud-controller-manager - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:cloud-controller-manager - subjects: - - apiGroup: "" - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system -kind: ConfigMap -metadata: - name: cloud-controller-manager-addon - namespace: default ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineHealthCheck -metadata: - name: ${CLUSTER_NAME}-mhc-0 - namespace: ${NAMESPACE} -spec: - clusterName: ${CLUSTER_NAME} - maxUnhealthy: 100% - nodeStartupTimeout: 30s - selector: - matchLabels: - cluster.x-k8s.io/control-plane: "" - mhc-test: fail - unhealthyConditions: - - status: "False" - timeout: 10s - type: e2e.remediation.condition diff --git a/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml deleted file mode 100644 index 4177c710..00000000 --- a/test/e2e/data/infrastructure-aws/cluster-template-md-remediation.yaml +++ /dev/null @@ -1,340 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - labels: - ccm: external -spec: - clusterNetwork: - pods: - cidrBlocks: - - 10.1.0.0/16 - services: - cidrBlocks: - - 10.152.183.0/24 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta2 - kind: CK8sControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: AWSCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: AWSCluster -metadata: - name: ${CLUSTER_NAME} -spec: - region: ${AWS_REGION} - sshKeyName: ${AWS_SSH_KEY_NAME} - bastion: - enabled: ${AWS_CREATE_BASTION} - controlPlaneLoadBalancer: - loadBalancerType: nlb - healthCheckProtocol: TCP - network: - cni: - cniIngressRules: - - description: microcluster - protocol: tcp - toPort: 2380 ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta2 -kind: CK8sControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - machineTemplate: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: AWSMachineTemplate - name: ${CLUSTER_NAME}-control-plane - spec: - nodeName: "{{ ds.meta_data.local_hostname }}" - channel: "1.31-classic/edge" - controlPlane: - cloudProvider: external - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: AWSMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane -spec: - template: - spec: - ami: - id: ${AWS_AMI_ID} - iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io - instanceType: ${AWS_CONTROL_PLANE_INSTANCE_TYPE} - publicIP: ${AWS_PUBLIC_IP} - sshKeyName: ${AWS_SSH_KEY_NAME} - rootVolume: - size: ${AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-worker-md-0 -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} - cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-worker-md-0 - template: - metadata: - labels: - cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-worker-md-0 - e2e.remediation.label: "" - spec: - version: ${KUBERNETES_VERSION} - clusterName: ${CLUSTER_NAME} - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 - kind: CK8sConfigTemplate - name: ${CLUSTER_NAME}-md-0 - infrastructureRef: - name: "${CLUSTER_NAME}-md-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: AWSMachineTemplate ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: AWSMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 -spec: - template: - spec: - ami: - id: ${AWS_AMI_ID} - iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io - instanceType: ${AWS_NODE_INSTANCE_TYPE} - publicIP: ${AWS_PUBLIC_IP} - sshKeyName: ${AWS_SSH_KEY_NAME} - rootVolume: - size: ${AWS_NODE_ROOT_VOLUME_SIZE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 -kind: CK8sConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 -spec: - template: - spec: - nodeName: "{{ ds.meta_data.local_hostname }}" - channel: "1.31-classic/edge" ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: crs-ccm -spec: - clusterSelector: - matchLabels: - ccm: external - resources: - - kind: ConfigMap - name: cloud-controller-manager-addon - strategy: ApplyOnce ---- -apiVersion: v1 -data: - aws-ccm-external.yaml: | - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: aws-cloud-controller-manager - namespace: kube-system - labels: - k8s-app: aws-cloud-controller-manager - spec: - selector: - matchLabels: - k8s-app: aws-cloud-controller-manager - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - k8s-app: aws-cloud-controller-manager - spec: - nodeSelector: - node-role.kubernetes.io/control-plane: "" - tolerations: - - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - effect: NoSchedule - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - serviceAccountName: cloud-controller-manager - containers: - - name: aws-cloud-controller-manager - image: ${AWS_CCM_IMAGE} - args: - - --v=2 - - --cloud-provider=aws - - --use-service-account-credentials=true - - --configure-cloud-routes=false - resources: - requests: - cpu: 200m - hostNetwork: true - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: cloud-controller-manager - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: cloud-controller-manager:apiserver-authentication-reader - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader - subjects: - - apiGroup: "" - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: system:cloud-controller-manager - rules: - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - '*' - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - services - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - services/status - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - get - - list - - watch - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - update - - watch - - apiGroups: - - "" - resources: - - endpoints - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - serviceaccounts/token - verbs: - - create - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: system:cloud-controller-manager - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:cloud-controller-manager - subjects: - - apiGroup: "" - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system -kind: ConfigMap -metadata: - name: cloud-controller-manager-addon - namespace: default ---- -# MachineHealthCheck object with -# - a selector that targets all the machines with label e2e.remediation.label="" -# - unhealthyConditions triggering remediation after 10s the condition is set -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineHealthCheck -metadata: - name: "${CLUSTER_NAME}-mhc-0" -spec: - clusterName: "${CLUSTER_NAME}" - maxUnhealthy: 100% - selector: - matchLabels: - e2e.remediation.label: "" - unhealthyConditions: - - type: e2e.remediation.condition - status: "False" - timeout: 10s diff --git a/test/e2e/kcp_remediation_test.go b/test/e2e/kcp_remediation_test.go index 439a8519..1cd6ddd9 100644 --- a/test/e2e/kcp_remediation_test.go +++ b/test/e2e/kcp_remediation_test.go @@ -27,6 +27,13 @@ import ( ) var _ = Describe("When testing KCP remediation", func() { + if clusterctl.DefaultInfrastructureProvider == "aws" { + // Skip the test for cloud provider as it is not supported + // See kubernetes.slack.com/archives/C8TSNPY4T/p1680525266510109 + // And github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4198 + return + } + capi_e2e.KCPRemediationSpec(ctx, func() capi_e2e.KCPRemediationSpecInput { return capi_e2e.KCPRemediationSpecInput{ E2EConfig: e2eConfig, @@ -34,6 +41,7 @@ var _ = Describe("When testing KCP remediation", func() { BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - InfrastructureProvider: ptr.To(clusterctl.DefaultInfrastructureProvider)} + InfrastructureProvider: ptr.To(clusterctl.DefaultInfrastructureProvider), + } }) }) diff --git a/test/e2e/md_remediation_test.go b/test/e2e/md_remediation_test.go index c50852a1..66970a94 100644 --- a/test/e2e/md_remediation_test.go +++ b/test/e2e/md_remediation_test.go @@ -34,6 +34,13 @@ import ( ) var _ = Describe("When testing MachineDeployment remediation", func() { + if clusterctl.DefaultInfrastructureProvider == "aws" { + // Skip the test for cloud provider as it is not supported + // See kubernetes.slack.com/archives/C8TSNPY4T/p1680525266510109 + // And github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4198 + return + } + var ( ctx = context.TODO() specName = "machine-deployment-remediation" From ac84c08d1b5c86bda566a624faeab9716393d8e5 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 17 Oct 2024 16:42:59 -0400 Subject: [PATCH 41/79] change run condition --- .github/workflows/e2e-deleteme.yaml | 71 ----------------------------- .github/workflows/e2e.yaml | 2 - 2 files changed, 73 deletions(-) delete mode 100644 .github/workflows/e2e-deleteme.yaml diff --git a/.github/workflows/e2e-deleteme.yaml b/.github/workflows/e2e-deleteme.yaml deleted file mode 100644 index b6c31c46..00000000 --- a/.github/workflows/e2e-deleteme.yaml +++ /dev/null @@ -1,71 +0,0 @@ -name: E2E Tests - -on: - pull_request: - -permissions: - contents: read - id-token: write - -jobs: - run-e2e-tests: - runs-on: [self-hosted, linux, X64, jammy, large] - name: Run E2E Tests - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Setup Go - uses: actions/setup-go@v2 - with: - go-version: ~1.22.0 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: us-east-2 - role-to-assume: arn:aws:iam::018302341396:role/GithubOIDC - role-duration-seconds: 3600 - - - name: Check out repo - uses: actions/checkout@v4 - - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 - # with: - # detached: true - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - # We run into rate limiting issues if we don't authenticate - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Install requirements - run: | - sudo apt install make wget - curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind - - - name: Increase inotify watches - run: | - # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files - sudo sysctl fs.inotify.max_user_watches=1048576 - sudo sysctl fs.inotify.max_user_instances=8192 - - - name: Build images - run: | - cd hack - ./build-e2e-images.sh - cd .. - make docker-build-e2e - - - name: Run e2e tests - run: | - E2E_INFRA=aws GINKGO_FOCUS="KCP remediation" make test-e2e - env: - GOPROXY: direct diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c9315b1a..ff1535ad 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2,8 +2,6 @@ name: E2E Tests on: pull_request: - branches: - - does-not-exist permissions: contents: read From 9045fc934813e02a4ef684e7dcd2fa0c6a76d41b Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 17 Oct 2024 16:44:41 -0400 Subject: [PATCH 42/79] add -y to apt install --- .github/workflows/e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index ff1535ad..7b1ea1ef 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -23,7 +23,7 @@ jobs: - name: Install requirements run: | sudo apt update - sudo apt install make docker-buildx + sudo apt install -y make docker-buildx sudo snap install go --classic --channel=1.22/stable sudo snap install kubectl --classic --channel=1.30/stable - name: Build provider images From c13b9c79e4568b8eb83441fe3ca24298b705ad93 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 17 Oct 2024 16:50:03 -0400 Subject: [PATCH 43/79] remove outdated files --- hack/ci-e2e-tests.sh | 217 ------------------------------- hack/install-clusterctl.sh | 17 --- hack/install-clusterctlawsadm.sh | 17 --- hack/write-provider-config.sh | 22 ---- 4 files changed, 273 deletions(-) delete mode 100755 hack/ci-e2e-tests.sh delete mode 100755 hack/install-clusterctl.sh delete mode 100644 hack/install-clusterctlawsadm.sh delete mode 100644 hack/write-provider-config.sh diff --git a/hack/ci-e2e-tests.sh b/hack/ci-e2e-tests.sh deleted file mode 100755 index f3b753e1..00000000 --- a/hack/ci-e2e-tests.sh +++ /dev/null @@ -1,217 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT enable -x as it will expose sensitive information in the logs. -# Enable debugging selectively using set -x and set +x around specific code blocks. -set -euo pipefail - -# This script is used to run e2e tests for the CK8s CAPI. -# It sets up an LXD container, installs the CK8s management cluster, and runs e2e tests. -# The goal is to test the CK8s provider with different infrastructure providers (e.g., AWS, Azure, GCP). Only AWS is supported for now. -# The script should be able to run on any Linux machine with LXD installed. - -# USAGE -# ./hack/ci-e2e-tests.sh [INFRA_PROVIDER] [CK8S_PROVIDER_VERSION] -# INFRA_PROVIDER: Optional. The infrastructure provider to use. Default is "aws". -# CK8S_PROVIDER_VERSION: Optional. The CK8s provider version to use. Default is "v0.1.2". - -readonly HACK_DIR="$(realpath $(dirname "${0}"))" -cd "$HACK_DIR" - -readonly INFRA_PROVIDER=${2:-aws} -readonly CK8S_PROVIDER_VERSION=${3:-v0.1.2} - -readonly LXD_CHANNEL="6.1/stable" -readonly LXC_IMAGE="ubuntu:22.04" -readonly K8S_PROFILE_URL="https://raw.githubusercontent.com/canonical/k8s-snap/main/tests/integration/lxd-profile.yaml" -readonly K8S_PROFILE_PATH="/tmp/k8s.profile" -readonly CONTAINER_NAME="k8s-test" - -# Utility function for printing errors to stderr -function error_exit { - printf "ERROR: %s\n" "$1" >&2 - return 1 -} - -function log_info { - printf "INFO: %s\n" "$1" -} - -# Check that all required environment variables are set -function check_required_env_vars { - log_info "Checking required environment variables..." - - local required_env_vars=() - - if [[ $INFRA_PROVIDER == "aws" ]]; then - required_env_vars+=("AWS_REGION" "AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY") - fi - - for var in "${required_env_vars[@]}"; do - if [ -z "${!var}" ]; then - error_exit "Missing required environment variable: $var" - fi - done -} - -function exec_in_container { - lxc exec $CONTAINER_NAME -- bash -c "$1" -} - -function setup_firewall { - log_info "Setting up firewall rules..." - - if sudo iptables -L DOCKER-USER; then - sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT - sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - fi -} - -# Install LXD snap -function install_lxd { - log_info "Installing LXD..." - - sudo snap install lxd --channel=$LXD_CHANNEL - sudo lxd waitready - sudo lxd init --auto - sudo usermod --append --groups lxd "$USER" -} - -# Create or ensure the k8s profile exists -function setup_lxd_profile { - log_info "Setting up LXD profile..." - - lxc profile show k8s || lxc profile create k8s - wget -q $K8S_PROFILE_URL -O $K8S_PROFILE_PATH - cat $K8S_PROFILE_PATH | lxc profile edit k8s - rm -f $K8S_PROFILE_PATH -} - -# Setup and configure the container -function setup_container { - log_info "Setting up LXD container..." - - lxc launch $LXC_IMAGE $CONTAINER_NAME -p default -p k8s - - # Wait for container to be ready to run commands - until exec_in_container true; do - sleep 1 - done - - exec_in_container "apt update && apt install -y snapd" - exec_in_container "systemctl start snapd" - exec_in_container "snap wait core seed.loaded" - - # Script is running from the hack directory, so push the entire directory to the container - lxc file push -r .. $CONTAINER_NAME/root/ >/dev/null -} - -function configure_container_env { - log_info "Configuring container environment..." - - if [[ $INFRA_PROVIDER == "aws" ]]; then - log_info "Configuring AWS credentials in container..." - - # Check for clusterawsadm binary - exec_in_container "which clusterawsadm" || error_exit "clusterawsadm binary not found in container" - - lxc config set $CONTAINER_NAME environment.AWS_REGION "$AWS_REGION" - lxc config set $CONTAINER_NAME environment.AWS_SECRET_ACCESS_KEY "$AWS_SECRET_ACCESS_KEY" - lxc config set $CONTAINER_NAME environment.AWS_ACCESS_KEY_ID "$AWS_ACCESS_KEY_ID" - - if [[ -z $AWS_SESSION_TOKEN ]]; then - log_info "AWS_SESSION_TOKEN not set. Skipping..." - else - lxc config set $CONTAINER_NAME environment.AWS_SESSION_TOKEN "$AWS_SESSION_TOKEN" - fi - - # This command can fail if the stack already exists, so we ignore the error - exec_in_container "clusterawsadm bootstrap iam create-cloudformation-stack" || true - - local aws_creds - aws_creds=$(lxc exec "$CONTAINER_NAME" -- bash -c "clusterawsadm bootstrap credentials encode-as-profile") - - echo "::add-mask::$aws_creds" # Mask the credentials in the Github CI logs. - lxc config set "$CONTAINER_NAME" environment.AWS_B64ENCODED_CREDENTIALS "$aws_creds" - fi -} - -# Main installation and configuration -function setup_management_cluster { - log_info "Setting up management cluster..." - exec_in_container "sudo snap install k8s --classic --edge" - exec_in_container "sudo snap install go --classic" - exec_in_container "mkdir -p /root/.kube" - exec_in_container "sudo k8s bootstrap" - exec_in_container "sudo k8s status --wait-ready" - exec_in_container "sudo k8s config > /root/.kube/config" -} - -function clone_repos { - log_info "Cloning CK8s and CAPI repositories..." - exec_in_container "git clone --depth 1 https://github.com/kubernetes-sigs/cluster-api-provider-aws /root/cluster-api-provider-aws" - exec_in_container "git clone --depth 1 https://github.com/kubernetes-sigs/cluster-api /root/cluster-api" -} - -# Transfer and execute scripts -function install_tools { - log_info "Installing tools in container..." - - tools=(install-clusterctl.sh) - packages=(make) - snaps=(kubectl) - - if [[ $INFRA_PROVIDER == "aws" ]]; then - tools+=(install-clusterctlawsadm.sh install-aws-nuke.sh) - fi - - for script in "${tools[@]}"; do - exec_in_container "chmod +x /root/cluster-api-k8s/hack/$script && /root/cluster-api-k8s/hack/$script" - done - - for package in "${packages[@]}"; do - exec_in_container "apt install -y $package" - done - - for snap in "${snaps[@]}"; do - exec_in_container "snap install $snap --classic" - done -} - -function init_clusterctl { - log_info "Initializing clusterctl with $INFRA_PROVIDER infrastructure and CK8s $CK8S_PROVIDER_VERSION..." - - configure_container_env # Ensures that the right environment variables are set in the container - - exec_in_container "chmod +x /root/cluster-api-k8s/hack/write-provider-config.sh" - exec_in_container "mkdir -p /root/.cluster-api" - exec_in_container "/root/cluster-api-k8s/hack/write-provider-config.sh /root/.cluster-api/clusterctl.yaml $CK8S_PROVIDER_VERSION" - - exec_in_container "clusterctl init -i $INFRA_PROVIDER -b ck8s:$CK8S_PROVIDER_VERSION -c ck8s:$CK8S_PROVIDER_VERSION --config /root/.cluster-api/clusterctl.yaml" -} - -function run_e2e_tests { - log_info "Running e2e tests..." - exec_in_container "cd /root/cluster-api-k8s && make USE_EXISTING_CLUSTER=true GINKGO_FOCUS=\"Workload cluster creation\" test-e2e" -} - -function main { - if [[ $INFRA_PROVIDER != "aws" ]]; then - error_exit "Unsupported infrastructure provider: $INFRA_PROVIDER" - exit 1 - fi - - check_required_env_vars - install_lxd - setup_lxd_profile - setup_firewall - setup_container - setup_management_cluster - clone_repos - install_tools - init_clusterctl - run_e2e_tests - - log_info "E2E tests completed successfully." -} - -main diff --git a/hack/install-clusterctl.sh b/hack/install-clusterctl.sh deleted file mode 100755 index 5be8b88a..00000000 --- a/hack/install-clusterctl.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Description: -# Installs clusterctl to /usr/local/bin -# -# Usage: -# $ install-clusterctl.sh - -set -o errexit -set -o nounset -set -o pipefail - -curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.1/clusterctl-linux-amd64 -o clusterctl -chmod +x ./clusterctl -sudo mv ./clusterctl /usr/local/bin - -clusterctl version diff --git a/hack/install-clusterctlawsadm.sh b/hack/install-clusterctlawsadm.sh deleted file mode 100644 index c838f69c..00000000 --- a/hack/install-clusterctlawsadm.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Description: -# Installs clusterawsadm to /usr/local/bin -# -# Usage: -# $ install-clusterawsadm.sh - -set -o errexit -set -o nounset -set -o pipefail - -curl -L https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.6.1/clusterawsadm-linux-amd64 -o clusterawsadm -chmod +x ./clusterawsadm -sudo mv ./clusterawsadm /usr/local/bin - -clusterawsadm version diff --git a/hack/write-provider-config.sh b/hack/write-provider-config.sh deleted file mode 100644 index 38bab5c7..00000000 --- a/hack/write-provider-config.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Description: -# Creates a clusterctl configuration file -# -# Usage: -# $ write-clusterctl-config.sh $output-file $version - -set -euo pipefail - -OUTPUT_FILE="$1" -VERSION="$2" - -cat <"$OUTPUT_FILE" -providers: - - name: "ck8s" - url: "https://github.com/canonical/cluster-api-k8s/releases/download/${VERSION}/bootstrap-components.yaml" - type: "BootstrapProvider" - - name: "ck8s" - url: "https://github.com/canonical/cluster-api-k8s/releases/download/${VERSION}/control-plane-components.yaml" - type: "ControlPlaneProvider" -EOF From 266d2f729d5a06788ddc75df4a800581b139721c Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 17 Oct 2024 17:45:09 -0400 Subject: [PATCH 44/79] refactor workflow --- .github/workflows/e2e.yaml | 48 ++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 7b1ea1ef..b07ae8ca 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -4,12 +4,13 @@ on: pull_request: permissions: + id-token: write contents: read jobs: - build-e2e-images: - name: Build & Run E2E Images - runs-on: [self-hosted, linux, X64, jammy, large] + build-provider-e2e-images: + name: Build Provider E2E Images + runs-on : [self-hosted, linux, X64, jammy, large] steps: - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -28,14 +29,41 @@ jobs: sudo snap install kubectl --classic --channel=1.30/stable - name: Build provider images run: sudo make docker-build-e2e - - name: Build k8s-snap images - working-directory: hack/ - run: | - ./build-e2e-images.sh - name: Save provider image run: | sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev sudo chmod 775 provider-images.tar + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: e2e-images + path: | + provider-images.tar + + build-k8s-snap-e2e-images: + name: Build K8s Snap E2E Images + if: false + runs-on: [self-hosted, linux, X64, jammy, large] + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + # We run into rate limiting issues if we don't authenticate + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Check out repo + uses: actions/checkout@v4 + - name: Install requirements + run: | + sudo apt update + sudo apt install -y make docker-buildx + sudo snap install go --classic --channel=1.22/stable + sudo snap install kubectl --classic --channel=1.30/stable + - name: Build k8s-snap images + working-directory: hack/ + run: | + ./build-e2e-images.sh - name: Save k8s-snap image run: | sudo docker save -o k8s-snap-image-old.tar k8s-snap:dev-old @@ -47,14 +75,13 @@ jobs: with: name: e2e-images path: | - provider-images.tar k8s-snap-image-old.tar k8s-snap-image-new.tar run-e2e-tests: name: Run E2E Tests runs-on: [self-hosted, linux, X64, jammy, xlarge] - needs: build-e2e-images + needs: [build-provider-e2e-images, build-k8s-snap-e2e-images] strategy: matrix: infra: @@ -93,10 +120,11 @@ jobs: - name: Load provider image run: sudo docker load -i provider-images.tar - name: Load k8s-snap old image + if: matrix.infra == 'docker' run: | sudo docker load -i k8s-snap-image-old.tar - name: Load k8s-snap new image - if: matrix.ginkgo_focus == 'Workload cluster upgrade' + if: matrix.infra == 'docker' && matrix.ginkgo_focus == 'Workload cluster upgrade' run: | sudo docker load -i k8s-snap-image-new.tar - name: Create docker network From 258499929eb1bf6bce5396b4f209d3f92e8588c5 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 17 Oct 2024 17:52:31 -0400 Subject: [PATCH 45/79] pull instead of build --- .github/workflows/e2e.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index b07ae8ca..69cd768e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -28,7 +28,12 @@ jobs: sudo snap install go --classic --channel=1.22/stable sudo snap install kubectl --classic --channel=1.30/stable - name: Build provider images - run: sudo make docker-build-e2e + #run: sudo make docker-build-e2e + run: | + docker pull ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:ci-test + docker tag ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:ci-test ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev + docker pull ghcr.io/canonical/cluster-api-k8s/controlplane-controller:ci-test + docker tag ghcr.io/canonical/cluster-api-k8s/controlplane-controller:ci-test ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev - name: Save provider image run: | sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev From 3b510f6c530935ab326abd6a0179fc9e5b5bc726 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 17 Oct 2024 17:55:29 -0400 Subject: [PATCH 46/79] dont require skipped job --- .github/workflows/e2e.yaml | 2 +- templates/aws/cluster-generated.yaml | 323 +++++++++++++++++++++++++++ 2 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 templates/aws/cluster-generated.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 69cd768e..ecfa1275 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -86,7 +86,7 @@ jobs: run-e2e-tests: name: Run E2E Tests runs-on: [self-hosted, linux, X64, jammy, xlarge] - needs: [build-provider-e2e-images, build-k8s-snap-e2e-images] + needs: [build-provider-e2e-images] strategy: matrix: infra: diff --git a/templates/aws/cluster-generated.yaml b/templates/aws/cluster-generated.yaml new file mode 100644 index 00000000..541ffd02 --- /dev/null +++ b/templates/aws/cluster-generated.yaml @@ -0,0 +1,323 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + ccm: external + name: k8s + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.1.0.0/16 + services: + cidrBlocks: + - 10.152.183.0/24 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: k8s-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSCluster + name: k8s +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSCluster +metadata: + name: k8s + namespace: default +spec: + bastion: + enabled: true + controlPlaneLoadBalancer: + healthCheckProtocol: TCP + network: + cni: + cniIngressRules: + - description: microcluster + protocol: tcp + toPort: 2380 + region: us-east-2 + sshKeyName: etienne +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: k8s-control-plane + namespace: default +spec: + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: k8s-control-plane + replicas: 1 + spec: + controlPlane: + cloudProvider: external + nodeName: '{{ ds.meta_data.local_hostname }}' + channel: "1.31/edge" + version: v1.31.0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: k8s-control-plane + namespace: default +spec: + template: + spec: + ami: + id: ami-01b139e6226d65e4f + iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io + instanceType: t3.large + publicIP: true + rootVolume: + size: 16 + sshKeyName: etienne +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: k8s-worker-md-0 + namespace: default +spec: + clusterName: k8s + replicas: 0 + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: k8s + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: CK8sConfigTemplate + name: k8s-md-0 + clusterName: k8s + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: k8s-md-0 + version: v1.31.0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: k8s-md-0 + namespace: default +spec: + template: + spec: + ami: + id: ami-01b139e6226d65e4f + iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io + instanceType: t3.large + publicIP: true + rootVolume: + size: 16 + sshKeyName: etienne +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: k8s-md-0 + namespace: default +spec: + template: + spec: + channel: "1.31/edge" + nodeName: '{{ ds.meta_data.local_hostname }}' +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 + args: + - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + - --configure-cloud-routes=false + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + name: cloud-controller-manager-addon + namespace: default From b85c08284b3ee1af945685166d7cb54e3830f1a4 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 17 Oct 2024 18:17:26 -0400 Subject: [PATCH 47/79] set awsb64encoded_credentials --- .github/workflows/e2e.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index ecfa1275..721102d5 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -141,6 +141,7 @@ jobs: sudo sysctl fs.inotify.max_user_watches=1048576 sudo sysctl fs.inotify.max_user_instances=8192 - name: Configure AWS Credentials + id: creds if: matrix.infra == 'aws' uses: aws-actions/configure-aws-credentials@v4 with: @@ -148,6 +149,17 @@ jobs: aws-region: us-east-2 role-to-assume: arn:aws:iam::018302341396:role/GithubOIDC role-duration-seconds: 3600 + output-credentials: true + - name: Set AWS Credentials as Environment Variables + if: matrix.infra == 'aws' + run: | + echo "AWS_ACCESS_KEY_ID=${{ steps.creds.outputs.aws-access-key-id }}" >> $GITHUB_ENV + echo "AWS_SECRET_ACCESS_KEY=${{ steps.creds.outputs.aws-secret-access-key }}" >> $GITHUB_ENV + echo "AWS_SESSION_TOKEN=${{ steps.creds.outputs.aws-session-token }}" >> $GITHUB_ENV + + AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile --region us-east-2) + echo "AWS_B64ENCODED_CREDENTIALS=$AWS_B64ENCODED_CREDENTIALS" >> $GITHUB_ENV 2>&1 + echo "::add-mask::$AWS_B64ENCODED_CREDENTIALS" - name: Run e2e tests if: ${{!(matrix.infra == 'aws' && (matrix.ginkgo_focus == 'KCP remediation' || matrix.ginkgo_focus == 'MachineDeployment remediation'))}} run: | From 044aa93cab10cdff9772afac81b0930b2d061c74 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 17 Oct 2024 18:18:18 -0400 Subject: [PATCH 48/79] only run one test for now --- .github/workflows/e2e.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 721102d5..3890a942 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -96,8 +96,8 @@ jobs: - "KCP remediation" - "MachineDeployment remediation" - "Workload cluster creation" - - "Workload cluster scaling" - - "Workload cluster upgrade" + #- "Workload cluster scaling" + #- "Workload cluster upgrade" # TODO(ben): Remove once all tests are running stable. fail-fast: false steps: From 70961773b161ff7f4fd5d1473dcbb27a9c7f2128 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 17 Oct 2024 18:22:18 -0400 Subject: [PATCH 49/79] install clusterawsadm --- .github/workflows/e2e.yaml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3890a942..73259c8e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -140,6 +140,13 @@ jobs: # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files sudo sysctl fs.inotify.max_user_watches=1048576 sudo sysctl fs.inotify.max_user_instances=8192 + - name: Install clusterawsadm + if: matrix.infra == 'aws' + run: | + curl -L https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.6.1/clusterawsadm-linux-amd64 -o clusterawsadm + chmod +x ./clusterawsadm + sudo mv ./clusterawsadm /usr/local/bin + clusterawsadm version - name: Configure AWS Credentials id: creds if: matrix.infra == 'aws' @@ -153,14 +160,16 @@ jobs: - name: Set AWS Credentials as Environment Variables if: matrix.infra == 'aws' run: | - echo "AWS_ACCESS_KEY_ID=${{ steps.creds.outputs.aws-access-key-id }}" >> $GITHUB_ENV - echo "AWS_SECRET_ACCESS_KEY=${{ steps.creds.outputs.aws-secret-access-key }}" >> $GITHUB_ENV - echo "AWS_SESSION_TOKEN=${{ steps.creds.outputs.aws-session-token }}" >> $GITHUB_ENV + export AWS_ACCESS_KEY_ID=${{ steps.creds.outputs.aws-access-key-id }} + export AWS_SECRET_ACCESS_KEY=${{ steps.creds.outputs.aws-secret-access-key }} + export AWS_SESSION_TOKEN=${{ steps.creds.outputs.aws-session-token }} AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile --region us-east-2) - echo "AWS_B64ENCODED_CREDENTIALS=$AWS_B64ENCODED_CREDENTIALS" >> $GITHUB_ENV 2>&1 + echo "AWS_B64ENCODED_CREDENTIALS=$AWS_B64ENCODED_CREDENTIALS" >> "$GITHUB_ENV" echo "::add-mask::$AWS_B64ENCODED_CREDENTIALS" - name: Run e2e tests if: ${{!(matrix.infra == 'aws' && (matrix.ginkgo_focus == 'KCP remediation' || matrix.ginkgo_focus == 'MachineDeployment remediation'))}} + env: + AWS_B64ENCODED_CREDENTIALS: ${{ env.AWS_B64ENCODED_CREDENTIALS }} run: | - sudo E2E_INFRA=${{matrix.infra}} GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e + sudo AWS_B64ENCODED_CREDENTIALS=$AWS_B64ENCODED_CREDENTIALS E2E_INFRA=${{matrix.infra}} GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e From a2a2573e71e338f52a514336d08ba4cf9bc4d2c8 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 17 Oct 2024 19:59:42 -0400 Subject: [PATCH 50/79] -E and change to create public IP --- .github/workflows/e2e.yaml | 4 +--- test/e2e/config/ck8s-aws.yaml | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 73259c8e..c0901b87 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -169,7 +169,5 @@ jobs: echo "::add-mask::$AWS_B64ENCODED_CREDENTIALS" - name: Run e2e tests if: ${{!(matrix.infra == 'aws' && (matrix.ginkgo_focus == 'KCP remediation' || matrix.ginkgo_focus == 'MachineDeployment remediation'))}} - env: - AWS_B64ENCODED_CREDENTIALS: ${{ env.AWS_B64ENCODED_CREDENTIALS }} run: | - sudo AWS_B64ENCODED_CREDENTIALS=$AWS_B64ENCODED_CREDENTIALS E2E_INFRA=${{matrix.infra}} GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e + sudo -E E2E_INFRA=${{matrix.infra}} GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index ee61c704..09e3d11f 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -89,9 +89,9 @@ variables: KIND_IMAGE_VERSION: "v1.30.0" AWS_CONTROL_PLANE_INSTANCE_TYPE: t3.large AWS_NODE_INSTANCE_TYPE: t3.large - AWS_PUBLIC_IP: false - AWS_CREATE_BASTION: false - AWS_SSH_KEY_NAME: "default" + AWS_PUBLIC_IP: true + AWS_CREATE_BASTION: true + AWS_SSH_KEY_NAME: "etienne" AWS_AMI_ID: "ami-01b139e6226d65e4f" AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE: 16 AWS_NODE_ROOT_VOLUME_SIZE: 16 From 8e7d62d32d25053bb0a08f742d7225eab684f68e Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Fri, 18 Oct 2024 09:11:11 -0400 Subject: [PATCH 51/79] improve aws-nuke cleanup --- .github/workflows/e2e.yaml | 18 ++++-- hack/aws-nuke-config.yaml | 119 ++++++++++--------------------------- hack/install-aws-nuke.sh | 10 ++-- 3 files changed, 50 insertions(+), 97 deletions(-) mode change 100644 => 100755 hack/install-aws-nuke.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c0901b87..404f011f 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -88,6 +88,7 @@ jobs: runs-on: [self-hosted, linux, X64, jammy, xlarge] needs: [build-provider-e2e-images] strategy: + max-parallel: 1 # Only one at a time because of AWS resource limitations (like maximum number of elastic ip's) matrix: infra: - "aws" @@ -115,8 +116,7 @@ jobs: sudo apt update sudo snap install go --classic --channel=1.22/stable sudo apt install make - sudo apt install docker-buildx - sudo snap install kubectl --classic --channel=1.30/stable + ./hack/install-aws-nuke.sh - name: Download artifacts uses: actions/download-artifact@v4 with: @@ -160,9 +160,13 @@ jobs: - name: Set AWS Credentials as Environment Variables if: matrix.infra == 'aws' run: | - export AWS_ACCESS_KEY_ID=${{ steps.creds.outputs.aws-access-key-id }} - export AWS_SECRET_ACCESS_KEY=${{ steps.creds.outputs.aws-secret-access-key }} - export AWS_SESSION_TOKEN=${{ steps.creds.outputs.aws-session-token }} + AWS_ACCESS_KEY_ID=${{ steps.creds.outputs.aws-access-key-id }} >> "$GITHUB_ENV" + AWS_SECRET_ACCESS_KEY=${{ steps.creds.outputs.aws-secret-access-key }} >> "$GITHUB_ENV" + AWS_SESSION_TOKEN=${{ steps.creds.outputs.aws-session-token }} >> "$GITHUB_ENV" + + export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID + export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY + export AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile --region us-east-2) echo "AWS_B64ENCODED_CREDENTIALS=$AWS_B64ENCODED_CREDENTIALS" >> "$GITHUB_ENV" @@ -171,3 +175,7 @@ jobs: if: ${{!(matrix.infra == 'aws' && (matrix.ginkgo_focus == 'KCP remediation' || matrix.ginkgo_focus == 'MachineDeployment remediation'))}} run: | sudo -E E2E_INFRA=${{matrix.infra}} GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e + - name: Cleanup AWS account + if: matrix.infra == 'aws' + run: | + aws-nuke run --config ./hack/aws-nuke-config.yaml --force --force-sleep 3 --no-dry-run diff --git a/hack/aws-nuke-config.yaml b/hack/aws-nuke-config.yaml index 758849b0..bd93620d 100644 --- a/hack/aws-nuke-config.yaml +++ b/hack/aws-nuke-config.yaml @@ -5,92 +5,37 @@ blocklist: - "999999999999" # A blocklist is required, but this is a placeholder accounts: - 018302341396: # AWS Account IDs are not considered sensitive information - filters: - EC2Instance: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - EC2SecurityGroup: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - EC2Volume: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - EC2InternetGateway: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - EC2NATGateway: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - EC2RouteTable: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - EC2Subnet: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - EC2VPC: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - EC2VPCEndpoint: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - EC2VPCEndpointServiceConfiguration: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - EC2ElasticIP: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - EC2NetworkInterface: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - ELBv2: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - ELBv2TargetGroup: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - S3Bucket: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - AutoScalingGroup: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true - AutoScalingLaunchConfiguration: - - property: "tag:sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - type: "glob" - invert: true + 018302341396: {} + +__global__: + - property: tag + type: glob + value: "sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" + resource-types: includes: - - EC2Instance - - EC2SecurityGroup - - EC2Volume - - EC2InternetGateway - - EC2NATGateway - - EC2RouteTable - - EC2Subnet - - EC2VPC - - EC2VPCEndpoint - - EC2VPCEndpointServiceConfiguration - - EC2ElasticIP - - EC2NetworkInterface - - ELBv2 - - ELBv2TargetGroup - - S3Bucket - - AutoScalingGroup - - AutoScalingLaunchConfiguration + - EC2Instance + - EC2SecurityGroup + - EC2Volume + - EC2InternetGateway + - EC2InternetGatewayAttachment + - EC2DHCPOption + - EC2NetworkACL + - EC2NATGateway + - EC2RouteTable + - EC2Subnet + - EC2Volume + - EC2VPC + - EC2VPCEndpoint + - EC2VPCEndpointServiceConfiguration + - EC2Address + - EC2NetworkInterface + - EC2VPCEndpointConnection + - EC2VPCPeeringConnection + - EC2EgressOnlyInternetGateway + - ELB + - ELBv2 + - ELBv2TargetGroup + - S3Bucket + - AutoScalingGroup + - AutoScalingLaunchConfiguration diff --git a/hack/install-aws-nuke.sh b/hack/install-aws-nuke.sh old mode 100644 new mode 100755 index 568bb7f8..1eaa6351 --- a/hack/install-aws-nuke.sh +++ b/hack/install-aws-nuke.sh @@ -10,10 +10,10 @@ set -o errexit set -o nounset set -o pipefail -curl -L https://github.com/rebuy-de/aws-nuke/releases/download/v2.25.0/aws-nuke-v2.25.0-linux-amd64.tar.gz -o aws-nuke-v2.25.0-linux-amd64.tar.gz -tar -xvf aws-nuke-v2.25.0-linux-amd64.tar.gz -C /tmp -rm aws-nuke-v2.25.0-linux-amd64.tar.gz -chmod +x /tmp/aws-nuke-v2.25.0-linux-amd64 -mv /tmp/aws-nuke-v2.25.0-linux-amd64 /usr/local/bin/aws-nuke +curl -L https://github.com/ekristen/aws-nuke/releases/download/v3.28.0/aws-nuke-v3.28.0-linux-amd64.tar.gz -o aws-nuke-v3.28.0-linux-amd64.tar.gz +tar -xvf aws-nuke-v3.28.0-linux-amd64.tar.gz -C /tmp +rm aws-nuke-v3.28.0-linux-amd64.tar.gz +chmod +x /tmp/aws-nuke +mv /tmp/aws-nuke /usr/local/bin/aws-nuke aws-nuke version From 2b3f85053441ecd820f7c45721fb04f0f04a18ba Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Fri, 18 Oct 2024 10:15:55 -0400 Subject: [PATCH 52/79] add kubectl --- .github/workflows/e2e.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 404f011f..933f90ba 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -115,6 +115,7 @@ jobs: run: | sudo apt update sudo snap install go --classic --channel=1.22/stable + sudo snap install kubectl --classic --channel 1.31/stable sudo apt install make ./hack/install-aws-nuke.sh - name: Download artifacts From 14fd2a5aa165e0fd062130e1e0e4d279b629475c Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Fri, 18 Oct 2024 11:25:35 -0400 Subject: [PATCH 53/79] add tmate --- .github/workflows/e2e.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 933f90ba..a88be169 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -111,6 +111,8 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Check out repo uses: actions/checkout@v4 + - name: Setup tmate session + uses: canonical/action-tmate@main - name: Install requirements run: | sudo apt update From 1fb4a3ac6fbe7c2adf383cf99577309736a55a62 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Fri, 18 Oct 2024 11:29:13 -0400 Subject: [PATCH 54/79] only run one test --- .github/workflows/e2e.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a88be169..07d56660 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -94,8 +94,8 @@ jobs: - "aws" #- "docker" ginkgo_focus: - - "KCP remediation" - - "MachineDeployment remediation" + #- "KCP remediation" + #- "MachineDeployment remediation" - "Workload cluster creation" #- "Workload cluster scaling" #- "Workload cluster upgrade" From e88a76a615072fb4bd025b2852e0f085072b8f7a Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Fri, 18 Oct 2024 12:23:19 -0400 Subject: [PATCH 55/79] detached --- .github/workflows/e2e.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 07d56660..9d5cab67 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -113,6 +113,8 @@ jobs: uses: actions/checkout@v4 - name: Setup tmate session uses: canonical/action-tmate@main + with: + detached: true - name: Install requirements run: | sudo apt update From 29f7eacc01c05270a62b7c359978f5a4cbbc5246 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Fri, 18 Oct 2024 14:30:15 -0400 Subject: [PATCH 56/79] change ssh key name --- test/e2e/config/ck8s-aws.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index 09e3d11f..38b27791 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -91,7 +91,7 @@ variables: AWS_NODE_INSTANCE_TYPE: t3.large AWS_PUBLIC_IP: true AWS_CREATE_BASTION: true - AWS_SSH_KEY_NAME: "etienne" + AWS_SSH_KEY_NAME: "default" AWS_AMI_ID: "ami-01b139e6226d65e4f" AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE: 16 AWS_NODE_ROOT_VOLUME_SIZE: 16 From c9254daf7a600b189d5a767c303368f478b5e046 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Fri, 18 Oct 2024 15:08:58 -0400 Subject: [PATCH 57/79] revert to 1.28.3 img and always cleanup --- .github/workflows/e2e.yaml | 2 +- test/e2e/config/ck8s-aws.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9d5cab67..621484c6 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -181,6 +181,6 @@ jobs: run: | sudo -E E2E_INFRA=${{matrix.infra}} GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e - name: Cleanup AWS account - if: matrix.infra == 'aws' + if: ${{ always() && matrix.infra == 'aws' }} run: | aws-nuke run --config ./hack/aws-nuke-config.yaml --force --force-sleep 3 --no-dry-run diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index 38b27791..85579c09 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -96,7 +96,7 @@ variables: AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE: 16 AWS_NODE_ROOT_VOLUME_SIZE: 16 AWS_REGION: "us-east-2" - AWS_CCM_IMAGE: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.30.0" + AWS_CCM_IMAGE: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3" # https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml#L203C1-L205C27 # There is some work to be done here on figuring out which experimental features # we want to enable/disable. From 879d15bd5fd571e320886bce0fa456c0acf525b3 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Fri, 18 Oct 2024 15:37:20 -0400 Subject: [PATCH 58/79] dont harcode namespace --- test/e2e/data/infrastructure-aws/cluster-template.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/data/infrastructure-aws/cluster-template.yaml b/test/e2e/data/infrastructure-aws/cluster-template.yaml index fc5b8561..c1fc3dff 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template.yaml @@ -313,4 +313,3 @@ data: kind: ConfigMap metadata: name: cloud-controller-manager-addon - namespace: default From a0cd0426b4eae941a775d6867a7a7c8439a4eb0d Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Fri, 18 Oct 2024 16:09:30 -0400 Subject: [PATCH 59/79] use explicit vpc, subnet, sg --- .../data/infrastructure-aws/cluster-template.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/e2e/data/infrastructure-aws/cluster-template.yaml b/test/e2e/data/infrastructure-aws/cluster-template.yaml index c1fc3dff..bd998fd3 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template.yaml @@ -33,6 +33,19 @@ spec: controlPlaneLoadBalancer: healthCheckProtocol: TCP network: + vpc: + id: vpc-09dc405df98c78806 + subnets: + - id: subnet-01d81a191cbc4129d + - id: subnet-08ec1c6ed394b2314 + - id: subnet-04c84a4861de91635 + - id: subnet-0fdbd3832cd20a307 + securityGroupOverrides: + bastion: sg-00cbb4ff658537143 + controlplane: sg-00cbb4ff658537143 + apiserver-lb: sg-00cbb4ff658537143 + node: sg-00cbb4ff658537143 + lb: sg-00cbb4ff658537143 cni: cniIngressRules: - description: microcluster From 9a60c49ea3ed072c1fcc5b7506c6f99e0022c270 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Fri, 18 Oct 2024 16:19:05 -0400 Subject: [PATCH 60/79] skip cleanup --- .github/workflows/e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 621484c6..9aea2581 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -181,6 +181,6 @@ jobs: run: | sudo -E E2E_INFRA=${{matrix.infra}} GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e - name: Cleanup AWS account - if: ${{ always() && matrix.infra == 'aws' }} + if: false run: | aws-nuke run --config ./hack/aws-nuke-config.yaml --force --force-sleep 3 --no-dry-run From e0e827551af48f0b67c3d4d465b31b5f2445631f Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Fri, 18 Oct 2024 17:18:05 -0400 Subject: [PATCH 61/79] try reconcile --- test/e2e/data/infrastructure-aws/cluster-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/data/infrastructure-aws/cluster-template.yaml b/test/e2e/data/infrastructure-aws/cluster-template.yaml index bd998fd3..9393bd8a 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template.yaml @@ -148,7 +148,7 @@ spec: resources: - kind: ConfigMap name: cloud-controller-manager-addon - strategy: ApplyOnce + strategy: Reconcile --- apiVersion: v1 data: From 8b2bfbc91b7ce7928a4c18ec4850abcc22871fbc Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Mon, 21 Oct 2024 14:11:25 -0400 Subject: [PATCH 62/79] reconcile -> applyonce --- test/e2e/data/infrastructure-aws/cluster-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/data/infrastructure-aws/cluster-template.yaml b/test/e2e/data/infrastructure-aws/cluster-template.yaml index 9393bd8a..bd998fd3 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template.yaml @@ -148,7 +148,7 @@ spec: resources: - kind: ConfigMap name: cloud-controller-manager-addon - strategy: Reconcile + strategy: ApplyOnce --- apiVersion: v1 data: From 2994cd22339f4570481a53cd13b2652bc23da3b9 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Mon, 21 Oct 2024 21:27:01 -0400 Subject: [PATCH 63/79] try juju --- .github/workflows/e2e.yaml | 3 ++- hack/juju-create-aws-instance.sh | 26 ++++++++++++++++++++++++++ hack/run-e2e-test.sh | 22 ++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100755 hack/juju-create-aws-instance.sh create mode 100755 hack/run-e2e-test.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9aea2581..3f0fc03c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -179,7 +179,8 @@ jobs: - name: Run e2e tests if: ${{!(matrix.infra == 'aws' && (matrix.ginkgo_focus == 'KCP remediation' || matrix.ginkgo_focus == 'MachineDeployment remediation'))}} run: | - sudo -E E2E_INFRA=${{matrix.infra}} GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e + #sudo -E E2E_INFRA=${{matrix.infra}} GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e + sudo -E ./hack/juju-create-aws-instance.sh - name: Cleanup AWS account if: false run: | diff --git a/hack/juju-create-aws-instance.sh b/hack/juju-create-aws-instance.sh new file mode 100755 index 00000000..5e766f6f --- /dev/null +++ b/hack/juju-create-aws-instance.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Description: +# Bootstraps a Juju cluster (1 machine) and installs all tools necessary +# to run the CAPI e2e tests on AWS, then runs the tests. +# +# Usage: +# $ juju-create-aws-instance.sh +# +# Assumptions: +# - These environment variables are set: +# - AWS_B64ENCODED_CREDENTIALS + +set -o nounset +set -o pipefail + +DIR="$(realpath $(dirname "${0}"))" + +# Bootstrap Juju +# Juju creates the instance that will host the management cluster +juju bootstrap aws/us-east-2 vimdiesel-aws --force --bootstrap-series jammy --bootstrap-constraints "arch=amd64" --model-default test-mode=true --model-default resource-tags=owner=vimdiesel --model-default automatically-retry-hooks=false --model-default 'logging-config==DEBUG' --model-default image-stream=daily --debug + +juju scp -m controller "$DIR"/run-e2e-test.sh 0:/home/ubuntu/run-e2e-test.sh + +#juju ssh --model controller 0 'sudo bash -s' <"$DIR"/run-e2e-test.sh +juju exec --model controller --unit controller/0 -- AWS_B64ENCODED_CREDENTIALS=${AWS_B64ENCODED_CREDENTIALS} /home/ubuntu/run-e2e-test.sh diff --git a/hack/run-e2e-test.sh b/hack/run-e2e-test.sh new file mode 100755 index 00000000..fc978fb2 --- /dev/null +++ b/hack/run-e2e-test.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +snap install go --classic --channel 1.22/stable +snap install kubectl --classic --channel 1.31/stable + +apt update +apt install -y docker.io docker-buildx make +systemctl enable --now docker + +curl -L https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.6.1/clusterawsadm-linux-amd64 -o clusterawsadm +chmod +x ./clusterawsadm +mv ./clusterawsadm /usr/local/bin +clusterawsadm version + +wget https://github.com/kubernetes-sigs/kind/releases/download/v0.24.0/kind-linux-amd64 -O /usr/local/bin/kind + +export KIND_EXPERIMENTAL_DOCKER_NETWORK=bridge +kind version + +git clone git@github.com:canonical/cluster-api-k8s.git /home/ubuntu/cluster-api-k8s && (cd /home/ubuntu/cluster-api-k8s || exit 1) + +sudo -E E2E_INFRA=aws GINKGO_FOCUS="Workload cluster creation" SKIP_RESOURCE_CLEANUP=true make test-e2e From 83f3b444b60ca1922d3027e87566324d198012f2 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 22 Oct 2024 08:01:47 -0400 Subject: [PATCH 64/79] use one file --- .github/workflows/e2e.yaml | 236 +++++++++++++++++++------------------ LICENSE | 223 ++++------------------------------- README.md | 91 +++++++++++++- hack/run-e2e-test.sh | 5 + yaml | 11 ++ 5 files changed, 245 insertions(+), 321 deletions(-) create mode 100644 yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3f0fc03c..4a5490f7 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -8,85 +8,85 @@ permissions: contents: read jobs: - build-provider-e2e-images: - name: Build Provider E2E Images - runs-on : [self-hosted, linux, X64, jammy, large] - steps: - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - # We run into rate limiting issues if we don't authenticate - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Check out repo - uses: actions/checkout@v4 - - name: Install requirements - run: | - sudo apt update - sudo apt install -y make docker-buildx - sudo snap install go --classic --channel=1.22/stable - sudo snap install kubectl --classic --channel=1.30/stable - - name: Build provider images - #run: sudo make docker-build-e2e - run: | - docker pull ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:ci-test - docker tag ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:ci-test ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev - docker pull ghcr.io/canonical/cluster-api-k8s/controlplane-controller:ci-test - docker tag ghcr.io/canonical/cluster-api-k8s/controlplane-controller:ci-test ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev - - name: Save provider image - run: | - sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev - sudo chmod 775 provider-images.tar - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: e2e-images - path: | - provider-images.tar + # build-provider-e2e-images: + # name: Build Provider E2E Images + # runs-on : [self-hosted, linux, X64, jammy, large] + # steps: + # - name: Login to GitHub Container Registry + # uses: docker/login-action@v3 + # with: + # # We run into rate limiting issues if we don't authenticate + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} + # - name: Check out repo + # uses: actions/checkout@v4 + # - name: Install requirements + # run: | + # sudo apt update + # sudo apt install -y make docker-buildx + # sudo snap install go --classic --channel=1.22/stable + # sudo snap install kubectl --classic --channel=1.30/stable + # - name: Build provider images + # #run: sudo make docker-build-e2e + # run: | + # docker pull ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:ci-test + # docker tag ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:ci-test ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev + # docker pull ghcr.io/canonical/cluster-api-k8s/controlplane-controller:ci-test + # docker tag ghcr.io/canonical/cluster-api-k8s/controlplane-controller:ci-test ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev + # - name: Save provider image + # run: | + # sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev + # sudo chmod 775 provider-images.tar + # - name: Upload artifacts + # uses: actions/upload-artifact@v4 + # with: + # name: e2e-images + # path: | + # provider-images.tar - build-k8s-snap-e2e-images: - name: Build K8s Snap E2E Images - if: false - runs-on: [self-hosted, linux, X64, jammy, large] - steps: - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - # We run into rate limiting issues if we don't authenticate - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Check out repo - uses: actions/checkout@v4 - - name: Install requirements - run: | - sudo apt update - sudo apt install -y make docker-buildx - sudo snap install go --classic --channel=1.22/stable - sudo snap install kubectl --classic --channel=1.30/stable - - name: Build k8s-snap images - working-directory: hack/ - run: | - ./build-e2e-images.sh - - name: Save k8s-snap image - run: | - sudo docker save -o k8s-snap-image-old.tar k8s-snap:dev-old - sudo docker save -o k8s-snap-image-new.tar k8s-snap:dev-new - sudo chmod 775 k8s-snap-image-old.tar - sudo chmod 775 k8s-snap-image-new.tar - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: e2e-images - path: | - k8s-snap-image-old.tar - k8s-snap-image-new.tar + # build-k8s-snap-e2e-images: + # name: Build K8s Snap E2E Images + # if: false + # runs-on: [self-hosted, linux, X64, jammy, large] + # steps: + # - name: Login to GitHub Container Registry + # uses: docker/login-action@v3 + # with: + # # We run into rate limiting issues if we don't authenticate + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} + # - name: Check out repo + # uses: actions/checkout@v4 + # - name: Install requirements + # run: | + # sudo apt update + # sudo apt install -y make docker-buildx + # sudo snap install go --classic --channel=1.22/stable + # sudo snap install kubectl --classic --channel=1.30/stable + # - name: Build k8s-snap images + # working-directory: hack/ + # run: | + # ./build-e2e-images.sh + # - name: Save k8s-snap image + # run: | + # sudo docker save -o k8s-snap-image-old.tar k8s-snap:dev-old + # sudo docker save -o k8s-snap-image-new.tar k8s-snap:dev-new + # sudo chmod 775 k8s-snap-image-old.tar + # sudo chmod 775 k8s-snap-image-new.tar + # - name: Upload artifacts + # uses: actions/upload-artifact@v4 + # with: + # name: e2e-images + # path: | + # k8s-snap-image-old.tar + # k8s-snap-image-new.tar run-e2e-tests: name: Run E2E Tests runs-on: [self-hosted, linux, X64, jammy, xlarge] - needs: [build-provider-e2e-images] + #needs: [build-provider-e2e-images] strategy: max-parallel: 1 # Only one at a time because of AWS resource limitations (like maximum number of elastic ip's) matrix: @@ -111,40 +111,40 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Check out repo uses: actions/checkout@v4 - - name: Setup tmate session - uses: canonical/action-tmate@main - with: - detached: true - - name: Install requirements - run: | - sudo apt update - sudo snap install go --classic --channel=1.22/stable - sudo snap install kubectl --classic --channel 1.31/stable - sudo apt install make - ./hack/install-aws-nuke.sh - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - name: e2e-images - path: . - - name: Load provider image - run: sudo docker load -i provider-images.tar - - name: Load k8s-snap old image - if: matrix.infra == 'docker' - run: | - sudo docker load -i k8s-snap-image-old.tar - - name: Load k8s-snap new image - if: matrix.infra == 'docker' && matrix.ginkgo_focus == 'Workload cluster upgrade' - run: | - sudo docker load -i k8s-snap-image-new.tar - - name: Create docker network - run: | - sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true - - name: Increase inotify watches - run: | - # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files - sudo sysctl fs.inotify.max_user_watches=1048576 - sudo sysctl fs.inotify.max_user_instances=8192 + # - name: Setup tmate session + # uses: canonical/action-tmate@main + # with: + # detached: true + # - name: Install requirements + # run: | + # sudo apt update + # sudo snap install go --classic --channel=1.22/stable + # sudo snap install kubectl --classic --channel 1.31/stable + # sudo apt install make + # ./hack/install-aws-nuke.sh + # - name: Download artifacts + # uses: actions/download-artifact@v4 + # with: + # name: e2e-images + # path: . + # - name: Load provider image + # run: sudo docker load -i provider-images.tar + # - name: Load k8s-snap old image + # if: matrix.infra == 'docker' + # run: | + # sudo docker load -i k8s-snap-image-old.tar + # - name: Load k8s-snap new image + # if: matrix.infra == 'docker' && matrix.ginkgo_focus == 'Workload cluster upgrade' + # run: | + # sudo docker load -i k8s-snap-image-new.tar + # - name: Create docker network + # run: | + # sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true + # - name: Increase inotify watches + # run: | + # # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files + # sudo sysctl fs.inotify.max_user_watches=1048576 + # sudo sysctl fs.inotify.max_user_instances=8192 - name: Install clusterawsadm if: matrix.infra == 'aws' run: | @@ -165,13 +165,15 @@ jobs: - name: Set AWS Credentials as Environment Variables if: matrix.infra == 'aws' run: | - AWS_ACCESS_KEY_ID=${{ steps.creds.outputs.aws-access-key-id }} >> "$GITHUB_ENV" - AWS_SECRET_ACCESS_KEY=${{ steps.creds.outputs.aws-secret-access-key }} >> "$GITHUB_ENV" - AWS_SESSION_TOKEN=${{ steps.creds.outputs.aws-session-token }} >> "$GITHUB_ENV" + #echo "AWS_ACCESS_KEY_ID=${{ steps.creds.outputs.aws-access-key-id }}" >> "$GITHUB_ENV" + #echo "AWS_SECRET_KEY_ID=${{ steps.creds.outputs.aws-secret-access-key }}" >> "$GITHUB_ENV" + #echo "AWS_SESSION_TOKEN=${{ steps.creds.outputs.aws-session-token }}" >> "$GITHUB_ENV" + #echo "AWS_REGION=us-east-2" >> "$GITHUB_ENV" - export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY - export AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN + #export AWS_ACCESS_KEY_ID=${{ steps.creds.outputs.aws-access-key-id }} + export AWS_SECRET_KEY_ID="${{ steps.creds.outputs.aws-secret-access-key }}" + echo "AWS_SECRET_KEY_ID=${{ steps.creds.outputs.aws-secret-access-key }}" >> "$GITHUB_ENV" + #export AWS_SESSION_TOKEN=${{ steps.creds.outputs.aws-session-token }} AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile --region us-east-2) echo "AWS_B64ENCODED_CREDENTIALS=$AWS_B64ENCODED_CREDENTIALS" >> "$GITHUB_ENV" @@ -179,8 +181,10 @@ jobs: - name: Run e2e tests if: ${{!(matrix.infra == 'aws' && (matrix.ginkgo_focus == 'KCP remediation' || matrix.ginkgo_focus == 'MachineDeployment remediation'))}} run: | - #sudo -E E2E_INFRA=${{matrix.infra}} GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e - sudo -E ./hack/juju-create-aws-instance.sh + sudo snap install juju --classic --channel 2.9/stable + juju bootstrap aws/us-east-2 vimdiesel-aws --force --bootstrap-series jammy --bootstrap-constraints "arch=amd64" --model-default test-mode=true --model-default resource-tags=owner=vimdiesel --model-default automatically-retry-hooks=false --model-default 'logging-config==DEBUG' --model-default image-stream=daily --debug + juju scp -m controller "$DIR"/run-e2e-test.sh 0:/home/ubuntu/run-e2e-test.sh + juju exec --model controller --unit controller/0 -- AWS_B64ENCODED_CREDENTIALS=${AWS_B64ENCODED_CREDENTIALS} /home/ubuntu/run-e2e-test.sh - name: Cleanup AWS account if: false run: | diff --git a/LICENSE b/LICENSE index 261eeb9e..440e09c2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,22 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +The MIT License (MIT) + +Copyright (c) 2016 reBuy reCommerce GmbH +Copyright (c) 2021 Erik Kristensen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 454ae375..8110bbd1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,90 @@ -# Cluster API Canonical Kubernetes +# aws-nuke -This repository contains bootstrap and control plane providers to deploy Canonical Kubernetes clusters using [Cluster API](https://github.com/kubernetes-sigs/cluster-api/blob/master/README.md). +[![license](https://img.shields.io/github/license/ekristen/aws-nuke.svg)](https://github.com/ekristen/aws-nuke/blob/main/LICENSE) +[![release](https://img.shields.io/github/release/ekristen/aws-nuke.svg)](https://github.com/ekristen/aws-nuke/releases) +[![Go Report Card](https://goreportcard.com/badge/github.com/ekristen/aws-nuke)](https://goreportcard.com/report/github.com/ekristen/aws-nuke) +[![Maintainability](https://api.codeclimate.com/v1/badges/bf05fb12c69f1ea7f257/maintainability)](https://codeclimate.com/github/ekristen/aws-nuke/maintainability) +![GitHub Downloads (all assets, all releases)](https://img.shields.io/github/downloads/ekristen/aws-nuke/total) +![GitHub Downloads (all assets, latest release)](https://img.shields.io/github/downloads/ekristen/aws-nuke/latest/total) -CABPCK (Cluster API bootstrap provider for Canonical Kubernetes) is responsible for generate cloud-init scripts for generate Machines such that they run Kubernetes nodes. This implementation uses [Canonical Kubernetes](https://github.com/canonical/k8s-snap) to deliver Kubernetes. -CACPCK (Cluster API control plane provider for Canonical Kubernetes) is responsible for managing the lifecycle of machines that host the control plane nodes of a Canonical Kubernetes cluster. + +## Overview + +Remove all resources from an AWS account. + +*aws-nuke* is stable, but it is likely that not all AWS resources are covered by it. Be encouraged to add missing +resources and create a Pull Request or to create an [Issue](https://github.com/ekristen/aws-nuke/issues/new). + +## What's New in Version 3 + +Version 3 is a rewrite of this tool using [libnuke](https://github.com/ekristen/libnuke) with a focus on improving a number of the outstanding things +that I couldn't get done with the original project without separating out the core code into a library. See Goals +below for more. + +This is not a comprehensive list, but here are some of the highlights: + +* New Feature: Signed Darwin Binaries for macOS +* New Feature: Published Homebrew Tap (ekristen/tap/aws-nuke@3) +* New Feature: Global Filters +* New Feature: Run Against All Enabled Regions +* New Feature: Explain Account and Explain Config Commands +* Upcoming Feature: Filter Groups (**in progress**) +* Breaking Change: `root` command no longer triggers the run, must use subcommand `run` (alias: `nuke`) +* Breaking Change: CloudFormation Stacks now support a hold and wait for parent deletion process +* Breaking Change: Nested CloudFormation Stacks are now eligible for deletion and no longer omitted +* Completely rewrote the core of the tool as a dedicated library [libnuke](https://github.com/ekristen/libnuke) + * This library has over 95% test coverage which makes iteration and new features easier to implement. +* Semantic Releases with notifications on issues / pull requests +* Context is passed throughout the entire library now, including the listing function and the removal function + * This is in preparation for supporting AWS SDK Go v2 +* New Resources +* Broke away from rebuy-de/aws-nuke project as a fork for reasons outlined in the history section + +### Goals + +- [x] Easier maintainability and bug fixing, see go report and code climate badges above +- [x] Adding additional tests around the core library +- [ ] Adding more tests around specific resource types +- [x] Adding additional resources and tooling to make adding resources easier +- [x] Adding documentation for adding resources and using the tool +- [ ] Consider adding DAG for dependencies between resource types and individual resources +- [ ] Support for AWS SDK Go v2 + +## Documentation + +All documentation is in the [docs/](docs) directory and is built using [Material for Mkdocs](https://squidfunk.github.io/mkdocs-material/). + +It is hosted at [https://ekristen.github.io/aws-nuke/](https://ekristen.github.io/aws-nuke/). + +## History of this Fork + +**Important:** this is a full fork of the original tool written by the folks over at [rebuy-de](https://github.com/rebuy-de). +This fork became necessary after attempting to make contributions and respond to issues to learn that the current +maintainers only have time to work on the project about once a month and while receptive to bringing in other +people to help maintain, made it clear it would take time. Considering the feedback cycle was already weeks on +initial communications, I had to make the hard decision to fork and maintain it. + +### libnuke + +I also needed a version of this tool for Azure and GCP, and initially I just copied and altered the code I needed for +Azure, but I didn't want to have to maintain multiple copies of the same code, so I decided to create +[libnuke](https://github.com/ekristen/libnuke) to abstract all the code that was common between the two tools and write +proper unit tests for it. + +## Attribution, License, and Copyright + +The rewrite of this tool to use [libnuke](https://github.com/ekristen/libnuke) would not have been possible without the +hard work that came before me on the original tool by the team and contributors over at [rebuy-de](https://github.com/rebuy-de) +and their original work on [rebuy-de/aws-nuke](https://github.com/rebuy-de/aws-nuke). + +This tool is licensed under the MIT license. See the [LICENSE](LICENSE) file for more information. The bulk of this +tool was rewritten to use [libnuke](https://github.com/ekristen/libnuke) which was in part originally sourced from +[rebuy-de/aws-nuke](https://github.com/rebuy-de/aws-nuke). + +## Contribute + +You can contribute to *aws-nuke* by forking this repository, making your changes and creating a Pull Request against +this repository. If you are unsure how to solve a problem or have other questions about a contributions, please create +a GitHub issue. + diff --git a/hack/run-e2e-test.sh b/hack/run-e2e-test.sh index fc978fb2..0e4489a1 100755 --- a/hack/run-e2e-test.sh +++ b/hack/run-e2e-test.sh @@ -17,6 +17,11 @@ wget https://github.com/kubernetes-sigs/kind/releases/download/v0.24.0/kind-linu export KIND_EXPERIMENTAL_DOCKER_NETWORK=bridge kind version +docker pull ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:ci-test +docker tag ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:ci-test ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev +docker pull ghcr.io/canonical/cluster-api-k8s/controlplane-controller:ci-test +docker tag ghcr.io/canonical/cluster-api-k8s/controlplane-controller:ci-test ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev + git clone git@github.com:canonical/cluster-api-k8s.git /home/ubuntu/cluster-api-k8s && (cd /home/ubuntu/cluster-api-k8s || exit 1) sudo -E E2E_INFRA=aws GINKGO_FOCUS="Workload cluster creation" SKIP_RESOURCE_CLEANUP=true make test-e2e diff --git a/yaml b/yaml new file mode 100644 index 00000000..976270bb --- /dev/null +++ b/yaml @@ -0,0 +1,11 @@ +Model Controller Cloud/Region Version SLA Timestamp +controller vimdiesel-aws aws/us-east-2 3.5.4 unsupported 20:32:01-04:00 + +App Version Status Scale Charm Channel Rev Exposed Message +controller active 1 juju-controller 3.5/stable 105 no + +Unit Workload Agent Machine Public address Ports Message +controller/0* active idle 0 18.224.64.247 + +Machine State Address Inst id Base AZ Message +0 started 18.224.64.247 i-0880751e552c2b6a1 ubuntu@22.04 us-east-2a running From 2f54e43f9e515d99ee621207e2fbf2f348d3a57a Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 22 Oct 2024 13:27:44 -0400 Subject: [PATCH 65/79] cleanup --- .github/workflows/e2e.yaml | 237 ++++++------------ Makefile | 6 +- hack/aws-nuke-config.yaml | 41 --- hack/juju-create-aws-instance.sh | 26 -- hack/run-e2e-test.sh | 27 -- test/e2e/config/ck8s-aws.yaml | 36 ++- .../infrastructure-aws/cluster-template.yaml | 13 - test/e2e/helpers.go | 2 +- yaml | 11 - 9 files changed, 98 insertions(+), 301 deletions(-) delete mode 100644 hack/aws-nuke-config.yaml delete mode 100755 hack/juju-create-aws-instance.sh delete mode 100755 hack/run-e2e-test.sh delete mode 100644 yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 4a5490f7..ddf1a02f 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -4,105 +4,72 @@ on: pull_request: permissions: - id-token: write contents: read jobs: - # build-provider-e2e-images: - # name: Build Provider E2E Images - # runs-on : [self-hosted, linux, X64, jammy, large] - # steps: - # - name: Login to GitHub Container Registry - # uses: docker/login-action@v3 - # with: - # # We run into rate limiting issues if we don't authenticate - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} - # - name: Check out repo - # uses: actions/checkout@v4 - # - name: Install requirements - # run: | - # sudo apt update - # sudo apt install -y make docker-buildx - # sudo snap install go --classic --channel=1.22/stable - # sudo snap install kubectl --classic --channel=1.30/stable - # - name: Build provider images - # #run: sudo make docker-build-e2e - # run: | - # docker pull ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:ci-test - # docker tag ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:ci-test ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev - # docker pull ghcr.io/canonical/cluster-api-k8s/controlplane-controller:ci-test - # docker tag ghcr.io/canonical/cluster-api-k8s/controlplane-controller:ci-test ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev - # - name: Save provider image - # run: | - # sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev - # sudo chmod 775 provider-images.tar - # - name: Upload artifacts - # uses: actions/upload-artifact@v4 - # with: - # name: e2e-images - # path: | - # provider-images.tar - - # build-k8s-snap-e2e-images: - # name: Build K8s Snap E2E Images - # if: false - # runs-on: [self-hosted, linux, X64, jammy, large] - # steps: - # - name: Login to GitHub Container Registry - # uses: docker/login-action@v3 - # with: - # # We run into rate limiting issues if we don't authenticate - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} - # - name: Check out repo - # uses: actions/checkout@v4 - # - name: Install requirements - # run: | - # sudo apt update - # sudo apt install -y make docker-buildx - # sudo snap install go --classic --channel=1.22/stable - # sudo snap install kubectl --classic --channel=1.30/stable - # - name: Build k8s-snap images - # working-directory: hack/ - # run: | - # ./build-e2e-images.sh - # - name: Save k8s-snap image - # run: | - # sudo docker save -o k8s-snap-image-old.tar k8s-snap:dev-old - # sudo docker save -o k8s-snap-image-new.tar k8s-snap:dev-new - # sudo chmod 775 k8s-snap-image-old.tar - # sudo chmod 775 k8s-snap-image-new.tar - # - name: Upload artifacts - # uses: actions/upload-artifact@v4 - # with: - # name: e2e-images - # path: | - # k8s-snap-image-old.tar - # k8s-snap-image-new.tar + build-e2e-images: + name: Build & Run E2E Images + runs-on: [self-hosted, linux, X64, jammy, large] + steps: + - + name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + # We run into rate limiting issues if we don't authenticate + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Check out repo + uses: actions/checkout@v4 + - name: Install requirements + run: | + sudo apt update + sudo snap install go --classic --channel=1.22/stable + sudo apt install make + sudo apt install docker-buildx + sudo snap install kubectl --classic --channel=1.30/stable + - name: Build provider images + run: sudo make docker-build-e2e + - name: Build k8s-snap images + working-directory: hack/ + run: | + ./build-e2e-images.sh + - name: Save provider image + run: | + sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev + sudo chmod 775 provider-images.tar + - name: Save k8s-snap image + run: | + sudo docker save -o k8s-snap-image-old.tar k8s-snap:dev-old + sudo docker save -o k8s-snap-image-new.tar k8s-snap:dev-new + sudo chmod 775 k8s-snap-image-old.tar + sudo chmod 775 k8s-snap-image-new.tar + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: e2e-images + path: | + provider-images.tar + k8s-snap-image-old.tar + k8s-snap-image-new.tar run-e2e-tests: name: Run E2E Tests runs-on: [self-hosted, linux, X64, jammy, xlarge] - #needs: [build-provider-e2e-images] + needs: build-e2e-images strategy: - max-parallel: 1 # Only one at a time because of AWS resource limitations (like maximum number of elastic ip's) matrix: - infra: - - "aws" - #- "docker" ginkgo_focus: - #- "KCP remediation" - #- "MachineDeployment remediation" + - "KCP remediation" + - "MachineDeployment remediation" - "Workload cluster creation" - #- "Workload cluster scaling" - #- "Workload cluster upgrade" + - "Workload cluster scaling" + - "Workload cluster upgrade" # TODO(ben): Remove once all tests are running stable. fail-fast: false steps: - - name: Login to GitHub Container Registry + - + name: Login to GitHub Container Registry uses: docker/login-action@v3 with: # We run into rate limiting issues if we don't authenticate @@ -111,81 +78,35 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Check out repo uses: actions/checkout@v4 - # - name: Setup tmate session - # uses: canonical/action-tmate@main - # with: - # detached: true - # - name: Install requirements - # run: | - # sudo apt update - # sudo snap install go --classic --channel=1.22/stable - # sudo snap install kubectl --classic --channel 1.31/stable - # sudo apt install make - # ./hack/install-aws-nuke.sh - # - name: Download artifacts - # uses: actions/download-artifact@v4 - # with: - # name: e2e-images - # path: . - # - name: Load provider image - # run: sudo docker load -i provider-images.tar - # - name: Load k8s-snap old image - # if: matrix.infra == 'docker' - # run: | - # sudo docker load -i k8s-snap-image-old.tar - # - name: Load k8s-snap new image - # if: matrix.infra == 'docker' && matrix.ginkgo_focus == 'Workload cluster upgrade' - # run: | - # sudo docker load -i k8s-snap-image-new.tar - # - name: Create docker network - # run: | - # sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true - # - name: Increase inotify watches - # run: | - # # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files - # sudo sysctl fs.inotify.max_user_watches=1048576 - # sudo sysctl fs.inotify.max_user_instances=8192 - - name: Install clusterawsadm - if: matrix.infra == 'aws' + - name: Install requirements run: | - curl -L https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.6.1/clusterawsadm-linux-amd64 -o clusterawsadm - chmod +x ./clusterawsadm - sudo mv ./clusterawsadm /usr/local/bin - clusterawsadm version - - name: Configure AWS Credentials - id: creds - if: matrix.infra == 'aws' - uses: aws-actions/configure-aws-credentials@v4 + sudo apt update + sudo snap install go --classic --channel=1.22/stable + sudo apt install make + sudo apt install docker-buildx + sudo snap install kubectl --classic --channel=1.30/stable + - name: Download artifacts + uses: actions/download-artifact@v4 with: - audience: sts.amazonaws.com - aws-region: us-east-2 - role-to-assume: arn:aws:iam::018302341396:role/GithubOIDC - role-duration-seconds: 3600 - output-credentials: true - - name: Set AWS Credentials as Environment Variables - if: matrix.infra == 'aws' + name: e2e-images + path: . + - name: Load provider image + run: sudo docker load -i provider-images.tar + - name: Load k8s-snap old image run: | - #echo "AWS_ACCESS_KEY_ID=${{ steps.creds.outputs.aws-access-key-id }}" >> "$GITHUB_ENV" - #echo "AWS_SECRET_KEY_ID=${{ steps.creds.outputs.aws-secret-access-key }}" >> "$GITHUB_ENV" - #echo "AWS_SESSION_TOKEN=${{ steps.creds.outputs.aws-session-token }}" >> "$GITHUB_ENV" - #echo "AWS_REGION=us-east-2" >> "$GITHUB_ENV" - - #export AWS_ACCESS_KEY_ID=${{ steps.creds.outputs.aws-access-key-id }} - export AWS_SECRET_KEY_ID="${{ steps.creds.outputs.aws-secret-access-key }}" - echo "AWS_SECRET_KEY_ID=${{ steps.creds.outputs.aws-secret-access-key }}" >> "$GITHUB_ENV" - #export AWS_SESSION_TOKEN=${{ steps.creds.outputs.aws-session-token }} - - AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile --region us-east-2) - echo "AWS_B64ENCODED_CREDENTIALS=$AWS_B64ENCODED_CREDENTIALS" >> "$GITHUB_ENV" - echo "::add-mask::$AWS_B64ENCODED_CREDENTIALS" - - name: Run e2e tests - if: ${{!(matrix.infra == 'aws' && (matrix.ginkgo_focus == 'KCP remediation' || matrix.ginkgo_focus == 'MachineDeployment remediation'))}} + sudo docker load -i k8s-snap-image-old.tar + - name: Load k8s-snap new image + if: matrix.ginkgo_focus == 'Workload cluster upgrade' + run: | + sudo docker load -i k8s-snap-image-new.tar + - name: Create docker network run: | - sudo snap install juju --classic --channel 2.9/stable - juju bootstrap aws/us-east-2 vimdiesel-aws --force --bootstrap-series jammy --bootstrap-constraints "arch=amd64" --model-default test-mode=true --model-default resource-tags=owner=vimdiesel --model-default automatically-retry-hooks=false --model-default 'logging-config==DEBUG' --model-default image-stream=daily --debug - juju scp -m controller "$DIR"/run-e2e-test.sh 0:/home/ubuntu/run-e2e-test.sh - juju exec --model controller --unit controller/0 -- AWS_B64ENCODED_CREDENTIALS=${AWS_B64ENCODED_CREDENTIALS} /home/ubuntu/run-e2e-test.sh - - name: Cleanup AWS account - if: false + sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true + - name: Increase inotify watches + run: | + # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files + sudo sysctl fs.inotify.max_user_watches=1048576 + sudo sysctl fs.inotify.max_user_instances=8192 + - name: Run e2e tests run: | - aws-nuke run --config ./hack/aws-nuke-config.yaml --force --force-sleep 3 --no-dry-run + sudo E2E_INFRA=docker GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e diff --git a/Makefile b/Makefile index bce01e26..e05f1b8b 100644 --- a/Makefile +++ b/Makefile @@ -99,7 +99,7 @@ GINKGO_NODES ?= 1 # GINKGO_NODES is the number of parallel nodes to run GINKGO_TIMEOUT ?= 2h GINKGO_POLL_PROGRESS_AFTER ?= 60m GINKGO_POLL_PROGRESS_INTERVAL ?= 5m -E2E_INFRA ?= aws +E2E_INFRA ?= docker E2E_CONF_FILE ?= $(TEST_DIR)/e2e/config/ck8s-$(E2E_INFRA).yaml SKIP_RESOURCE_CLEANUP ?= false USE_EXISTING_CLUSTER ?= false @@ -408,7 +408,3 @@ $(CONTROLLER_GEN): ## Build controller-gen from tools folder. $(CONVERSION_GEN): ## Build conversion-gen from tools folder. GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) k8s.io/code-generator/cmd/conversion-gen $(CONVERSION_GEN_BIN) $(CONVERSION_GEN_VER) - -.PHONY: nuke -nuke: - aws-nuke -c ~/.config/aws-nuke/nuke-config.yaml --no-dry-run diff --git a/hack/aws-nuke-config.yaml b/hack/aws-nuke-config.yaml deleted file mode 100644 index bd93620d..00000000 --- a/hack/aws-nuke-config.yaml +++ /dev/null @@ -1,41 +0,0 @@ -regions: - - us-east-2 - -blocklist: - - "999999999999" # A blocklist is required, but this is a placeholder - -accounts: - 018302341396: {} - -__global__: - - property: tag - type: glob - value: "sigs.k8s.io/cluster-api-provider-aws/cluster/capick8s-*" - -resource-types: - includes: - - EC2Instance - - EC2SecurityGroup - - EC2Volume - - EC2InternetGateway - - EC2InternetGatewayAttachment - - EC2DHCPOption - - EC2NetworkACL - - EC2NATGateway - - EC2RouteTable - - EC2Subnet - - EC2Volume - - EC2VPC - - EC2VPCEndpoint - - EC2VPCEndpointServiceConfiguration - - EC2Address - - EC2NetworkInterface - - EC2VPCEndpointConnection - - EC2VPCPeeringConnection - - EC2EgressOnlyInternetGateway - - ELB - - ELBv2 - - ELBv2TargetGroup - - S3Bucket - - AutoScalingGroup - - AutoScalingLaunchConfiguration diff --git a/hack/juju-create-aws-instance.sh b/hack/juju-create-aws-instance.sh deleted file mode 100755 index 5e766f6f..00000000 --- a/hack/juju-create-aws-instance.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Description: -# Bootstraps a Juju cluster (1 machine) and installs all tools necessary -# to run the CAPI e2e tests on AWS, then runs the tests. -# -# Usage: -# $ juju-create-aws-instance.sh -# -# Assumptions: -# - These environment variables are set: -# - AWS_B64ENCODED_CREDENTIALS - -set -o nounset -set -o pipefail - -DIR="$(realpath $(dirname "${0}"))" - -# Bootstrap Juju -# Juju creates the instance that will host the management cluster -juju bootstrap aws/us-east-2 vimdiesel-aws --force --bootstrap-series jammy --bootstrap-constraints "arch=amd64" --model-default test-mode=true --model-default resource-tags=owner=vimdiesel --model-default automatically-retry-hooks=false --model-default 'logging-config==DEBUG' --model-default image-stream=daily --debug - -juju scp -m controller "$DIR"/run-e2e-test.sh 0:/home/ubuntu/run-e2e-test.sh - -#juju ssh --model controller 0 'sudo bash -s' <"$DIR"/run-e2e-test.sh -juju exec --model controller --unit controller/0 -- AWS_B64ENCODED_CREDENTIALS=${AWS_B64ENCODED_CREDENTIALS} /home/ubuntu/run-e2e-test.sh diff --git a/hack/run-e2e-test.sh b/hack/run-e2e-test.sh deleted file mode 100755 index 0e4489a1..00000000 --- a/hack/run-e2e-test.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -snap install go --classic --channel 1.22/stable -snap install kubectl --classic --channel 1.31/stable - -apt update -apt install -y docker.io docker-buildx make -systemctl enable --now docker - -curl -L https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.6.1/clusterawsadm-linux-amd64 -o clusterawsadm -chmod +x ./clusterawsadm -mv ./clusterawsadm /usr/local/bin -clusterawsadm version - -wget https://github.com/kubernetes-sigs/kind/releases/download/v0.24.0/kind-linux-amd64 -O /usr/local/bin/kind - -export KIND_EXPERIMENTAL_DOCKER_NETWORK=bridge -kind version - -docker pull ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:ci-test -docker tag ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:ci-test ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev -docker pull ghcr.io/canonical/cluster-api-k8s/controlplane-controller:ci-test -docker tag ghcr.io/canonical/cluster-api-k8s/controlplane-controller:ci-test ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev - -git clone git@github.com:canonical/cluster-api-k8s.git /home/ubuntu/cluster-api-k8s && (cd /home/ubuntu/cluster-api-k8s || exit 1) - -sudo -E E2E_INFRA=aws GINKGO_FOCUS="Workload cluster creation" SKIP_RESOURCE_CLEANUP=true make test-e2e diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index 85579c09..d023c6b6 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -91,7 +91,7 @@ variables: AWS_NODE_INSTANCE_TYPE: t3.large AWS_PUBLIC_IP: true AWS_CREATE_BASTION: true - AWS_SSH_KEY_NAME: "default" + AWS_SSH_KEY_NAME: "etienne" AWS_AMI_ID: "ami-01b139e6226d65e4f" AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE: 16 AWS_NODE_ROOT_VOLUME_SIZE: 16 @@ -101,30 +101,28 @@ variables: # There is some work to be done here on figuring out which experimental features # we want to enable/disable. EXP_CLUSTER_RESOURCE_SET: "true" - EXP_RUNTIME_SDK: "true" EXP_MACHINE_SET_PREFLIGHT_CHECKS: "false" - EXP_MACHINE_POOL: "true" CLUSTER_TOPOLOGY: "true" CAPA_LOGLEVEL: "4" intervals: # Ref: https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml - default/wait-machines: [ "35m", "10s" ] - default/wait-cluster: [ "35m", "10s" ] - default/wait-control-plane: [ "35m", "10s" ] - default/wait-worker-nodes: [ "20m", "10s" ] - conformance/wait-control-plane: [ "35m", "10s" ] - conformance/wait-worker-nodes: [ "35m", "10s" ] - default/wait-controllers: [ "5m", "10s" ] - default/wait-delete-cluster: [ "20m", "10s" ] - default/wait-machine-upgrade: [ "35m", "10s" ] - default/wait-contolplane-upgrade: [ "40m", "10s" ] - default/wait-machine-status: [ "25m", "10s" ] - default/wait-failed-machine-status: [ "2m", "10s" ] + default/wait-machines: [ "10m", "10s" ] + default/wait-cluster: [ "10m", "10s" ] + default/wait-control-plane: [ "10m", "10s" ] + default/wait-worker-nodes: [ "10m", "10s" ] + conformance/wait-control-plane: [ "10m", "10s" ] + conformance/wait-worker-nodes: [ "10m", "10s" ] + default/wait-controllers: [ "10m", "10s" ] + default/wait-delete-cluster: [ "10m", "10s" ] + default/wait-machine-upgrade: [ "10m", "10s" ] + default/wait-contolplane-upgrade: [ "10m", "10s" ] + default/wait-machine-status: [ "10m", "10s" ] + default/wait-failed-machine-status: [ "10m", "10s" ] default/wait-infra-subnets: [ "5m", "30s" ] - default/wait-machine-pool-nodes: [ "40m", "10s" ] - default/wait-machine-pool-upgrade: [ "50m", "10s" ] - default/wait-create-identity: [ "1m", "10s" ] + default/wait-machine-pool-nodes: [ "10m", "10s" ] + default/wait-machine-pool-upgrade: [ "10m", "10s" ] + default/wait-create-identity: [ "3m", "10s" ] default/wait-job: [ "10m", "10s" ] - default/wait-deployment-ready: [ "5m", "10s" ] + default/wait-deployment-ready: [ "10m", "10s" ] default/wait-loadbalancer-ready: [ "5m", "30s" ] diff --git a/test/e2e/data/infrastructure-aws/cluster-template.yaml b/test/e2e/data/infrastructure-aws/cluster-template.yaml index bd998fd3..c1fc3dff 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template.yaml @@ -33,19 +33,6 @@ spec: controlPlaneLoadBalancer: healthCheckProtocol: TCP network: - vpc: - id: vpc-09dc405df98c78806 - subnets: - - id: subnet-01d81a191cbc4129d - - id: subnet-08ec1c6ed394b2314 - - id: subnet-04c84a4861de91635 - - id: subnet-0fdbd3832cd20a307 - securityGroupOverrides: - bastion: sg-00cbb4ff658537143 - controlplane: sg-00cbb4ff658537143 - apiserver-lb: sg-00cbb4ff658537143 - node: sg-00cbb4ff658537143 - lb: sg-00cbb4ff658537143 cni: cniIngressRules: - description: microcluster diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 23a521b7..4f3284d1 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -525,7 +525,7 @@ type WaitForControlPlaneAndMachinesReadyInput struct { ControlPlane *controlplanev1.CK8sControlPlane } -// WaitForControlPlaneAndMachinesReady waits for a KThreeControlPlane object to be ready (all the machine provisioned and one node ready). +// WaitForControlPlaneAndMachinesReady waits for a CK8sControlPlane object to be ready (all the machine provisioned and one node ready). func WaitForControlPlaneAndMachinesReady(ctx context.Context, input WaitForControlPlaneAndMachinesReadyInput, intervals ...interface{}) { Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForControlPlaneReady") Expect(input.GetLister).ToNot(BeNil(), "Invalid argument. input.GetLister can't be nil when calling WaitForControlPlaneReady") diff --git a/yaml b/yaml deleted file mode 100644 index 976270bb..00000000 --- a/yaml +++ /dev/null @@ -1,11 +0,0 @@ -Model Controller Cloud/Region Version SLA Timestamp -controller vimdiesel-aws aws/us-east-2 3.5.4 unsupported 20:32:01-04:00 - -App Version Status Scale Charm Channel Rev Exposed Message -controller active 1 juju-controller 3.5/stable 105 no - -Unit Workload Agent Machine Public address Ports Message -controller/0* active idle 0 18.224.64.247 - -Machine State Address Inst id Base AZ Message -0 started 18.224.64.247 i-0880751e552c2b6a1 ubuntu@22.04 us-east-2a running From f374d8f600912256e452473580836652098dda0c Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 22 Oct 2024 13:31:13 -0400 Subject: [PATCH 66/79] revert --- LICENSE | 223 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 201 insertions(+), 22 deletions(-) diff --git a/LICENSE b/LICENSE index 440e09c2..261eeb9e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,22 +1,201 @@ -The MIT License (MIT) - -Copyright (c) 2016 reBuy reCommerce GmbH -Copyright (c) 2021 Erik Kristensen - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From e08976e2ea5367db8a212d1b3293d7adc4b6e9c0 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 22 Oct 2024 13:31:43 -0400 Subject: [PATCH 67/79] revert --- templates/aws/cluster-generated.yaml | 323 --------------------------- 1 file changed, 323 deletions(-) delete mode 100644 templates/aws/cluster-generated.yaml diff --git a/templates/aws/cluster-generated.yaml b/templates/aws/cluster-generated.yaml deleted file mode 100644 index 541ffd02..00000000 --- a/templates/aws/cluster-generated.yaml +++ /dev/null @@ -1,323 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - ccm: external - name: k8s - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 10.1.0.0/16 - services: - cidrBlocks: - - 10.152.183.0/24 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta2 - kind: CK8sControlPlane - name: k8s-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: AWSCluster - name: k8s ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: AWSCluster -metadata: - name: k8s - namespace: default -spec: - bastion: - enabled: true - controlPlaneLoadBalancer: - healthCheckProtocol: TCP - network: - cni: - cniIngressRules: - - description: microcluster - protocol: tcp - toPort: 2380 - region: us-east-2 - sshKeyName: etienne ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta2 -kind: CK8sControlPlane -metadata: - name: k8s-control-plane - namespace: default -spec: - machineTemplate: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: AWSMachineTemplate - name: k8s-control-plane - replicas: 1 - spec: - controlPlane: - cloudProvider: external - nodeName: '{{ ds.meta_data.local_hostname }}' - channel: "1.31/edge" - version: v1.31.0 ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: AWSMachineTemplate -metadata: - name: k8s-control-plane - namespace: default -spec: - template: - spec: - ami: - id: ami-01b139e6226d65e4f - iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io - instanceType: t3.large - publicIP: true - rootVolume: - size: 16 - sshKeyName: etienne ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: k8s-worker-md-0 - namespace: default -spec: - clusterName: k8s - replicas: 0 - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: k8s - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 - kind: CK8sConfigTemplate - name: k8s-md-0 - clusterName: k8s - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: AWSMachineTemplate - name: k8s-md-0 - version: v1.31.0 ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: AWSMachineTemplate -metadata: - name: k8s-md-0 - namespace: default -spec: - template: - spec: - ami: - id: ami-01b139e6226d65e4f - iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io - instanceType: t3.large - publicIP: true - rootVolume: - size: 16 - sshKeyName: etienne ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 -kind: CK8sConfigTemplate -metadata: - name: k8s-md-0 - namespace: default -spec: - template: - spec: - channel: "1.31/edge" - nodeName: '{{ ds.meta_data.local_hostname }}' ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: crs-ccm - namespace: default -spec: - clusterSelector: - matchLabels: - ccm: external - resources: - - kind: ConfigMap - name: cloud-controller-manager-addon - strategy: ApplyOnce ---- -apiVersion: v1 -data: - aws-ccm-external.yaml: | - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: aws-cloud-controller-manager - namespace: kube-system - labels: - k8s-app: aws-cloud-controller-manager - spec: - selector: - matchLabels: - k8s-app: aws-cloud-controller-manager - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - k8s-app: aws-cloud-controller-manager - spec: - nodeSelector: - node-role.kubernetes.io/control-plane: "" - tolerations: - - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - effect: NoSchedule - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - serviceAccountName: cloud-controller-manager - containers: - - name: aws-cloud-controller-manager - image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 - args: - - --v=2 - - --cloud-provider=aws - - --use-service-account-credentials=true - - --configure-cloud-routes=false - resources: - requests: - cpu: 200m - hostNetwork: true - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: cloud-controller-manager - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: cloud-controller-manager:apiserver-authentication-reader - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader - subjects: - - apiGroup: "" - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: system:cloud-controller-manager - rules: - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - '*' - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - services - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - services/status - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - get - - list - - watch - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - update - - watch - - apiGroups: - - "" - resources: - - endpoints - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - serviceaccounts/token - verbs: - - create - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: system:cloud-controller-manager - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:cloud-controller-manager - subjects: - - apiGroup: "" - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system -kind: ConfigMap -metadata: - name: cloud-controller-manager-addon - namespace: default From 4b8c46f65d68c080e76ed9e930a54db92faac549 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 22 Oct 2024 13:32:28 -0400 Subject: [PATCH 68/79] revert --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e05f1b8b..b1de8246 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ KUSTOMIZE := $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER) # Ginkgo TEST_DIR := $(shell pwd)/test ARTIFACTS ?= $(shell pwd)/_artifacts -GINKGO_FOCUS ?= Workload cluster creation +GINKGO_FOCUS ?= GINKGO_SKIP ?= GINKGO_NODES ?= 1 # GINKGO_NODES is the number of parallel nodes to run # when running the e2e tests, 1 means no parallelism From 5d58ef3d7d36c88e7892b52e0e38880824e32b85 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 22 Oct 2024 13:42:31 -0400 Subject: [PATCH 69/79] update readme --- test/e2e/README.md | 50 +++++++++++++--------------------------------- 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index f092ebfd..802b0b43 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -29,6 +29,18 @@ This is useful if you want to use a cluster managed by Tilt. make USE_EXISTING_CLUSTER=true test-e2e ``` +### Run e2e tests on AWS + +To run the tests on AWS you will need to set the AWS_B64ENCODED_CREDENTIALS environment variable. + +Then, you can run: + +```shell +make E2E_INFRA=aws test-e2e +``` + +**Note**: The remediation tests do not pass on cloud providers. We suggest excluding them from the test run. See https://kubernetes.slack.com/archives/C8TSNPY4T/p1680525266510109. + ### Running the tests with Tilt This section explains how to run the E2E tests on AWS using a management cluster run by Tilt. @@ -52,15 +64,9 @@ enable_providers: - aws - ck8s-bootstrap - ck8s-control-plane -kustomize_substitutions: - EXP_CLUSTER_RESOURCE_SET: "true" # todo(eac): revisit these exp. features - EXP_MACHINE_POOL: "true" - CAPA_EKS_IAM: "false" - CAPA_EKS_ADD_ROLES: "false" - AWS_B64ENCODED_CREDENTIALS: "" ``` -Tilt will know how to run the aws provider controllers because the `cluster-api-provider-aws` repository has a `tilt-provider.json` file at it's root. Canonical Kubernetes also provides this file at the root of the repository. The CK8s provider names, ck8s-bootstrap and ck8s-control-plane, are defined in CK8's `tilt-provider.yaml` file. +Tilt will know how to run the aws provider controllers because the `cluster-api-provider-aws` repository has a `tilt-provider.yaml` file at it's root. Canonical Kubernetes also provides this file at the root of the repository. The CK8s provider names, ck8s-bootstrap and ck8s-control-plane, are defined in CK8's `tilt-provider.yaml` file. Next, you have to customize the variables that will be substituted into the cluster templates applied by the tests (these are under `test/e2e/data/infrastructure-aws`). You can customize the variables in the `test/e2e/config/ck8s-aws.yaml` file under the `variables` key. @@ -74,35 +80,7 @@ The test framework tries it's best to cleanup resources after a test suite, but cloud resources are left over. This can be very problematic especially if you run the tests multiple times while iterating on development (see [Cluster API Book - Tear down](https://cluster-api.sigs.k8s.io/developer/e2e#tear-down)). -You can use a tool like [aws-nuke](https://github.com/rebuy-de/aws-nuke) to cleanup your AWS account after a test. Here is a config. you can use that should cover most resources: - -```yaml -regions: - - us-east-2 - -account-blocklist: - - "" - -accounts: - "": {} - -resource-types: - targets: - - EC2Instance - - EC2SecurityGroup - - EC2Volume - - EC2InternetGateway - - EC2NATGateway - - EC2RouteTable - - EC2Subnet - - EC2VPC - - EC2VPCEndpoint - - EC2VPCEndpointServiceConfiguration - - EC2ElasticIP - - EC2NetworkInterface - - ELBv2 - - ELBv2TargetGroup -``` +You can use a tool like [aws-nuke](https://github.com/eriksten/aws-nuke) to cleanup your AWS account after a test. ## Develop an e2e test From 9f396b3c31644e16805ef7dbd5b428b0ad8bd643 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 22 Oct 2024 13:43:22 -0400 Subject: [PATCH 70/79] revert --- README.md | 91 +++---------------------------------------------------- 1 file changed, 4 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index 8110bbd1..454ae375 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,7 @@ -# aws-nuke +# Cluster API Canonical Kubernetes -[![license](https://img.shields.io/github/license/ekristen/aws-nuke.svg)](https://github.com/ekristen/aws-nuke/blob/main/LICENSE) -[![release](https://img.shields.io/github/release/ekristen/aws-nuke.svg)](https://github.com/ekristen/aws-nuke/releases) -[![Go Report Card](https://goreportcard.com/badge/github.com/ekristen/aws-nuke)](https://goreportcard.com/report/github.com/ekristen/aws-nuke) -[![Maintainability](https://api.codeclimate.com/v1/badges/bf05fb12c69f1ea7f257/maintainability)](https://codeclimate.com/github/ekristen/aws-nuke/maintainability) -![GitHub Downloads (all assets, all releases)](https://img.shields.io/github/downloads/ekristen/aws-nuke/total) -![GitHub Downloads (all assets, latest release)](https://img.shields.io/github/downloads/ekristen/aws-nuke/latest/total) +This repository contains bootstrap and control plane providers to deploy Canonical Kubernetes clusters using [Cluster API](https://github.com/kubernetes-sigs/cluster-api/blob/master/README.md). +CABPCK (Cluster API bootstrap provider for Canonical Kubernetes) is responsible for generate cloud-init scripts for generate Machines such that they run Kubernetes nodes. This implementation uses [Canonical Kubernetes](https://github.com/canonical/k8s-snap) to deliver Kubernetes. - -## Overview - -Remove all resources from an AWS account. - -*aws-nuke* is stable, but it is likely that not all AWS resources are covered by it. Be encouraged to add missing -resources and create a Pull Request or to create an [Issue](https://github.com/ekristen/aws-nuke/issues/new). - -## What's New in Version 3 - -Version 3 is a rewrite of this tool using [libnuke](https://github.com/ekristen/libnuke) with a focus on improving a number of the outstanding things -that I couldn't get done with the original project without separating out the core code into a library. See Goals -below for more. - -This is not a comprehensive list, but here are some of the highlights: - -* New Feature: Signed Darwin Binaries for macOS -* New Feature: Published Homebrew Tap (ekristen/tap/aws-nuke@3) -* New Feature: Global Filters -* New Feature: Run Against All Enabled Regions -* New Feature: Explain Account and Explain Config Commands -* Upcoming Feature: Filter Groups (**in progress**) -* Breaking Change: `root` command no longer triggers the run, must use subcommand `run` (alias: `nuke`) -* Breaking Change: CloudFormation Stacks now support a hold and wait for parent deletion process -* Breaking Change: Nested CloudFormation Stacks are now eligible for deletion and no longer omitted -* Completely rewrote the core of the tool as a dedicated library [libnuke](https://github.com/ekristen/libnuke) - * This library has over 95% test coverage which makes iteration and new features easier to implement. -* Semantic Releases with notifications on issues / pull requests -* Context is passed throughout the entire library now, including the listing function and the removal function - * This is in preparation for supporting AWS SDK Go v2 -* New Resources -* Broke away from rebuy-de/aws-nuke project as a fork for reasons outlined in the history section - -### Goals - -- [x] Easier maintainability and bug fixing, see go report and code climate badges above -- [x] Adding additional tests around the core library -- [ ] Adding more tests around specific resource types -- [x] Adding additional resources and tooling to make adding resources easier -- [x] Adding documentation for adding resources and using the tool -- [ ] Consider adding DAG for dependencies between resource types and individual resources -- [ ] Support for AWS SDK Go v2 - -## Documentation - -All documentation is in the [docs/](docs) directory and is built using [Material for Mkdocs](https://squidfunk.github.io/mkdocs-material/). - -It is hosted at [https://ekristen.github.io/aws-nuke/](https://ekristen.github.io/aws-nuke/). - -## History of this Fork - -**Important:** this is a full fork of the original tool written by the folks over at [rebuy-de](https://github.com/rebuy-de). -This fork became necessary after attempting to make contributions and respond to issues to learn that the current -maintainers only have time to work on the project about once a month and while receptive to bringing in other -people to help maintain, made it clear it would take time. Considering the feedback cycle was already weeks on -initial communications, I had to make the hard decision to fork and maintain it. - -### libnuke - -I also needed a version of this tool for Azure and GCP, and initially I just copied and altered the code I needed for -Azure, but I didn't want to have to maintain multiple copies of the same code, so I decided to create -[libnuke](https://github.com/ekristen/libnuke) to abstract all the code that was common between the two tools and write -proper unit tests for it. - -## Attribution, License, and Copyright - -The rewrite of this tool to use [libnuke](https://github.com/ekristen/libnuke) would not have been possible without the -hard work that came before me on the original tool by the team and contributors over at [rebuy-de](https://github.com/rebuy-de) -and their original work on [rebuy-de/aws-nuke](https://github.com/rebuy-de/aws-nuke). - -This tool is licensed under the MIT license. See the [LICENSE](LICENSE) file for more information. The bulk of this -tool was rewritten to use [libnuke](https://github.com/ekristen/libnuke) which was in part originally sourced from -[rebuy-de/aws-nuke](https://github.com/rebuy-de/aws-nuke). - -## Contribute - -You can contribute to *aws-nuke* by forking this repository, making your changes and creating a Pull Request against -this repository. If you are unsure how to solve a problem or have other questions about a contributions, please create -a GitHub issue. - +CACPCK (Cluster API control plane provider for Canonical Kubernetes) is responsible for managing the lifecycle of machines that host the control plane nodes of a Canonical Kubernetes cluster. From d4e2a1b7d2b0feba64b09b302376c2e5f6721e9c Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 22 Oct 2024 13:46:46 -0400 Subject: [PATCH 71/79] use skip and update readme --- test/e2e/README.md | 8 +++++++- test/e2e/kcp_remediation_test.go | 7 +++---- test/e2e/md_remediation_test.go | 7 +++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 802b0b43..5c617bf5 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -39,7 +39,13 @@ Then, you can run: make E2E_INFRA=aws test-e2e ``` -**Note**: The remediation tests do not pass on cloud providers. We suggest excluding them from the test run. See https://kubernetes.slack.com/archives/C8TSNPY4T/p1680525266510109. +Note: The remediation tests currently do not pass on cloud providers. We recommend excluding these tests from your test runs. + +For more information, please refer to the following: + +[Kubernetes Slack Discussion](kubernetes.slack.com/archives/C8TSNPY4T/p1680525266510109) + +[Github Issue #4198](github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4198) ### Running the tests with Tilt diff --git a/test/e2e/kcp_remediation_test.go b/test/e2e/kcp_remediation_test.go index 1cd6ddd9..3c3b220e 100644 --- a/test/e2e/kcp_remediation_test.go +++ b/test/e2e/kcp_remediation_test.go @@ -27,11 +27,10 @@ import ( ) var _ = Describe("When testing KCP remediation", func() { + // See kubernetes.slack.com/archives/C8TSNPY4T/p1680525266510109 + // And github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4198 if clusterctl.DefaultInfrastructureProvider == "aws" { - // Skip the test for cloud provider as it is not supported - // See kubernetes.slack.com/archives/C8TSNPY4T/p1680525266510109 - // And github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4198 - return + Skip("Skipping KCP remediation test for AWS") } capi_e2e.KCPRemediationSpec(ctx, func() capi_e2e.KCPRemediationSpecInput { diff --git a/test/e2e/md_remediation_test.go b/test/e2e/md_remediation_test.go index 66970a94..5f40620e 100644 --- a/test/e2e/md_remediation_test.go +++ b/test/e2e/md_remediation_test.go @@ -34,11 +34,10 @@ import ( ) var _ = Describe("When testing MachineDeployment remediation", func() { + // See kubernetes.slack.com/archives/C8TSNPY4T/p1680525266510109 + // And github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4198 if clusterctl.DefaultInfrastructureProvider == "aws" { - // Skip the test for cloud provider as it is not supported - // See kubernetes.slack.com/archives/C8TSNPY4T/p1680525266510109 - // And github.com/kubernetes-sigs/cluster-api-provider-aws/issues/4198 - return + Skip("Skipping KCP remediation test for AWS") } var ( From 46f42ce83f176dc801850d8e6f2a3c1ce69b6989 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 22 Oct 2024 13:52:53 -0400 Subject: [PATCH 72/79] revert --- hack/install-aws-nuke.sh | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100755 hack/install-aws-nuke.sh diff --git a/hack/install-aws-nuke.sh b/hack/install-aws-nuke.sh deleted file mode 100755 index 1eaa6351..00000000 --- a/hack/install-aws-nuke.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Description: -# Installs aws-nuke to /usr/local/bin -# -# Usage: -# $ install-aws-nuke.sh - -set -o errexit -set -o nounset -set -o pipefail - -curl -L https://github.com/ekristen/aws-nuke/releases/download/v3.28.0/aws-nuke-v3.28.0-linux-amd64.tar.gz -o aws-nuke-v3.28.0-linux-amd64.tar.gz -tar -xvf aws-nuke-v3.28.0-linux-amd64.tar.gz -C /tmp -rm aws-nuke-v3.28.0-linux-amd64.tar.gz -chmod +x /tmp/aws-nuke -mv /tmp/aws-nuke /usr/local/bin/aws-nuke - -aws-nuke version From a9d0f2babad0bdc21358fe983b4cfcdd47689ce7 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 22 Oct 2024 13:53:40 -0400 Subject: [PATCH 73/79] revert --- templates/aws/template-variables.rc | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/templates/aws/template-variables.rc b/templates/aws/template-variables.rc index 48313d8a..5939f9da 100644 --- a/templates/aws/template-variables.rc +++ b/templates/aws/template-variables.rc @@ -1,19 +1,20 @@ # Kubernetes cluster configuration -export KUBERNETES_VERSION=v1.30.0 -export CONTROL_PLANE_MACHINE_COUNT= # e.g. 1 -export WORKER_MACHINE_COUNT= +export KUBERNETES_VERSION=v1.31.0 +export CONTROL_PLANE_MACHINE_COUNT=3 +export WORKER_MACHINE_COUNT=3 # AWS region -export AWS_REGION="" # e.g. "us-east-2" +export AWS_REGION="eu-central-1" # AWS machine configuration -export AWS_CREATE_BASTION= # e.g. "true" -export AWS_PUBLIC_IP= # e.g. "true" -export AWS_CONTROL_PLANE_INSTANCE_TYPE= # e.g. "t3.large" -export AWS_NODE_INSTANCE_TYPE= # e.g. "t3.large" -export AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE= # in Gi, e.g. 16 -export AWS_NODE_ROOT_VOLUME_SIZE= # in Gi, e.g. 16 -export AWS_SSH_KEY_NAME= # e.g. "default" -export AWS_AMI_ID= # e.g. "ami-0ad50e72a79228704" +export AWS_CREATE_BASTION="true" +export AWS_PUBLIC_IP="true" +export AWS_CONTROL_PLANE_INSTANCE_TYPE="t3.large" +export AWS_NODE_INSTANCE_TYPE="t3.large" +export AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE=16 +export AWS_NODE_ROOT_VOLUME_SIZE=16 +export AWS_SSH_KEY_NAME="default" +# List upstream AMIs with clusterawsadm ami list --owner-id 819546954734 +export AWS_AMI_ID="ami-027b534ab5d0b4886" export AWS_CCM_IMAGE=registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 From 71dfed01f5fd9c28f2ca7ca2fa24bb785e6f3fa7 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Tue, 22 Oct 2024 21:51:26 -0400 Subject: [PATCH 74/79] increase timeouts --- test/e2e/config/ck8s-aws.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index d023c6b6..7e7600c2 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -107,22 +107,22 @@ variables: intervals: # Ref: https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/test/e2e/data/e2e_conf.yaml - default/wait-machines: [ "10m", "10s" ] - default/wait-cluster: [ "10m", "10s" ] - default/wait-control-plane: [ "10m", "10s" ] - default/wait-worker-nodes: [ "10m", "10s" ] - conformance/wait-control-plane: [ "10m", "10s" ] - conformance/wait-worker-nodes: [ "10m", "10s" ] - default/wait-controllers: [ "10m", "10s" ] - default/wait-delete-cluster: [ "10m", "10s" ] - default/wait-machine-upgrade: [ "10m", "10s" ] - default/wait-contolplane-upgrade: [ "10m", "10s" ] - default/wait-machine-status: [ "10m", "10s" ] - default/wait-failed-machine-status: [ "10m", "10s" ] + default/wait-machines: [ "35m", "10s" ] + default/wait-cluster: [ "35m", "10s" ] + default/wait-control-plane: [ "35m", "10s" ] + default/wait-worker-nodes: [ "35m", "10s" ] + conformance/wait-control-plane: [ "35m", "10s" ] + conformance/wait-worker-nodes: [ "35m", "10s" ] + default/wait-controllers: [ "35m", "10s" ] + default/wait-delete-cluster: [ "35m", "10s" ] + default/wait-machine-upgrade: [ "35m", "10s" ] + default/wait-contolplane-upgrade: [ "35m", "10s" ] + default/wait-machine-status: [ "35m", "10s" ] + default/wait-failed-machine-status: [ "35m", "10s" ] default/wait-infra-subnets: [ "5m", "30s" ] - default/wait-machine-pool-nodes: [ "10m", "10s" ] - default/wait-machine-pool-upgrade: [ "10m", "10s" ] + default/wait-machine-pool-nodes: [ "35m", "10s" ] + default/wait-machine-pool-upgrade: [ "35m", "10s" ] default/wait-create-identity: [ "3m", "10s" ] - default/wait-job: [ "10m", "10s" ] - default/wait-deployment-ready: [ "10m", "10s" ] + default/wait-job: [ "35m", "10s" ] + default/wait-deployment-ready: [ "35m", "10s" ] default/wait-loadbalancer-ready: [ "5m", "30s" ] From dd4f5c96a3aa18aa573e4961c1c8ef9cb2889aec Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Wed, 23 Oct 2024 09:49:33 -0400 Subject: [PATCH 75/79] bump to 1.8.4 --- .vscode/settings.json | 22 ++++++++++++++++++++++ test/e2e/config/ck8s-docker.yaml | 8 ++++---- 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..c67b5ae6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,22 @@ +{ + "workbench.colorCustomizations": { + "activityBar.activeBackground": "#9bb2f6", + "activityBar.background": "#9bb2f6", + "activityBar.foreground": "#15202b", + "activityBar.inactiveForeground": "#15202b99", + "activityBarBadge.background": "#ea154b", + "activityBarBadge.foreground": "#e7e7e7", + "commandCenter.border": "#15202b99", + "sash.hoverBorder": "#9bb2f6", + "statusBar.background": "#6c8ef2", + "statusBar.foreground": "#15202b", + "statusBarItem.hoverBackground": "#3d6aee", + "statusBarItem.remoteBackground": "#6c8ef2", + "statusBarItem.remoteForeground": "#15202b", + "titleBar.activeBackground": "#6c8ef2", + "titleBar.activeForeground": "#15202b", + "titleBar.inactiveBackground": "#6c8ef299", + "titleBar.inactiveForeground": "#15202b99" + }, + "peacock.remoteColor": "#6c8ef2" +} diff --git a/test/e2e/config/ck8s-docker.yaml b/test/e2e/config/ck8s-docker.yaml index 8dba00e0..a5175491 100644 --- a/test/e2e/config/ck8s-docker.yaml +++ b/test/e2e/config/ck8s-docker.yaml @@ -15,8 +15,8 @@ providers: - name: cluster-api type: CoreProvider versions: - - name: v1.6.2 - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.2/core-components.yaml + - name: v1.8.4 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.4/core-components.yaml type: url files: - sourcePath: "../data/shared/v1beta1/metadata.yaml" @@ -28,8 +28,8 @@ providers: versions: # By default, will use the latest version defined in ../data/shared/v1beta1/metadata.yaml # to init the management cluster - - name: v1.6.2 # used during e2e-test - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.2/infrastructure-components-development.yaml + - name: v1.8.4 # used during e2e-test + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.4/infrastructure-components-development.yaml type: url files: - sourcePath: "../data/shared/v1beta1/metadata.yaml" From 533bb54ab5609b3b927441f8c53105d9745623b8 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Wed, 23 Oct 2024 11:19:34 -0400 Subject: [PATCH 76/79] bump to 1.8.4 --- test/e2e/config/ck8s-aws.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index 7e7600c2..17576ba2 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -17,8 +17,8 @@ providers: - name: cluster-api type: CoreProvider versions: - - name: v1.8.3 - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.3/core-components.yaml + - name: v1.8.4 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.4/core-components.yaml type: url contract: v1beta1 files: From 20680962f4f3a0f72eed68ebd8d6b9ae91f421ef Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Wed, 23 Oct 2024 13:31:21 -0400 Subject: [PATCH 77/79] review comments --- .github/workflows/e2e.yaml | 2 +- test/e2e/config/ck8s-aws.yaml | 2 +- test/e2e/data/infrastructure-aws/cluster-template.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index ddf1a02f..2fbbd3a3 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -109,4 +109,4 @@ jobs: sudo sysctl fs.inotify.max_user_instances=8192 - name: Run e2e tests run: | - sudo E2E_INFRA=docker GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e + sudo GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index 17576ba2..495b0d20 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -91,7 +91,7 @@ variables: AWS_NODE_INSTANCE_TYPE: t3.large AWS_PUBLIC_IP: true AWS_CREATE_BASTION: true - AWS_SSH_KEY_NAME: "etienne" + AWS_SSH_KEY_NAME: "default" AWS_AMI_ID: "ami-01b139e6226d65e4f" AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE: 16 AWS_NODE_ROOT_VOLUME_SIZE: 16 diff --git a/test/e2e/data/infrastructure-aws/cluster-template.yaml b/test/e2e/data/infrastructure-aws/cluster-template.yaml index c1fc3dff..66290338 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template.yaml @@ -52,7 +52,7 @@ spec: name: ${CLUSTER_NAME}-control-plane spec: nodeName: "{{ ds.meta_data.local_hostname }}" - channel: "1.31-classic/edge" + channel: "{KUBERNETES_VERSION}-classic/edge" controlPlane: cloudProvider: external replicas: ${CONTROL_PLANE_MACHINE_COUNT} @@ -122,7 +122,7 @@ spec: template: spec: nodeName: "{{ ds.meta_data.local_hostname }}" - channel: "1.31-classic/edge" + channel: "{KUBERNETES_VERSION}-classic/edge" --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet From 2ef33892553d701f230cf4efde3a5024e4d7b74c Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Wed, 23 Oct 2024 13:31:33 -0400 Subject: [PATCH 78/79] revert --- .vscode/settings.json | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index c67b5ae6..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "workbench.colorCustomizations": { - "activityBar.activeBackground": "#9bb2f6", - "activityBar.background": "#9bb2f6", - "activityBar.foreground": "#15202b", - "activityBar.inactiveForeground": "#15202b99", - "activityBarBadge.background": "#ea154b", - "activityBarBadge.foreground": "#e7e7e7", - "commandCenter.border": "#15202b99", - "sash.hoverBorder": "#9bb2f6", - "statusBar.background": "#6c8ef2", - "statusBar.foreground": "#15202b", - "statusBarItem.hoverBackground": "#3d6aee", - "statusBarItem.remoteBackground": "#6c8ef2", - "statusBarItem.remoteForeground": "#15202b", - "titleBar.activeBackground": "#6c8ef2", - "titleBar.activeForeground": "#15202b", - "titleBar.inactiveBackground": "#6c8ef299", - "titleBar.inactiveForeground": "#15202b99" - }, - "peacock.remoteColor": "#6c8ef2" -} From 374cc2d05041c20b7e918d46db5f9f9cd8d5264d Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Wed, 23 Oct 2024 13:55:04 -0400 Subject: [PATCH 79/79] forgot $ --- test/e2e/data/infrastructure-aws/cluster-template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/data/infrastructure-aws/cluster-template.yaml b/test/e2e/data/infrastructure-aws/cluster-template.yaml index 66290338..e0a96714 100644 --- a/test/e2e/data/infrastructure-aws/cluster-template.yaml +++ b/test/e2e/data/infrastructure-aws/cluster-template.yaml @@ -52,7 +52,7 @@ spec: name: ${CLUSTER_NAME}-control-plane spec: nodeName: "{{ ds.meta_data.local_hostname }}" - channel: "{KUBERNETES_VERSION}-classic/edge" + channel: "${KUBERNETES_VERSION}-classic/edge" controlPlane: cloudProvider: external replicas: ${CONTROL_PLANE_MACHINE_COUNT} @@ -122,7 +122,7 @@ spec: template: spec: nodeName: "{{ ds.meta_data.local_hostname }}" - channel: "{KUBERNETES_VERSION}-classic/edge" + channel: "${KUBERNETES_VERSION}-classic/edge" --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet