From a4be61c55a1aee9000095dbbc34cf753b5bdfb50 Mon Sep 17 00:00:00 2001 From: Jonathan Meier Date: Tue, 27 Feb 2024 18:17:31 -0500 Subject: [PATCH] wip --- projects/aws/bottlerocket-bootstrap/CHECKSUMS | 8 +- projects/aws/bottlerocket-bootstrap/go.mod | 89 ++- projects/aws/bottlerocket-bootstrap/go.sum | 147 +++- .../pkg/constants/constants.go | 682 ++++++++++++++++++ .../pkg/kubeadm/controlplane_join.go | 2 + .../pkg/kubeadm/etcd/etcd.go | 640 ++++++++++++++++ .../pkg/kubeadm/testdata/manifests/etcd | 83 +++ .../kubeadm/testdata/manifests/kube-apiserver | 133 ++++ .../pkg/kubeadm/testdata/manifests/kube-vip | 52 ++ .../pkg/kubeadm/utils.go | 38 + .../pkg/kubeadm/utils_test.go | 17 + .../bottlerocket-bootstrap/pkg/utils/pods.go | 2 +- 12 files changed, 1845 insertions(+), 48 deletions(-) create mode 100644 projects/aws/bottlerocket-bootstrap/pkg/constants/constants.go create mode 100644 projects/aws/bottlerocket-bootstrap/pkg/kubeadm/etcd/etcd.go create mode 100644 projects/aws/bottlerocket-bootstrap/pkg/kubeadm/testdata/manifests/etcd create mode 100644 projects/aws/bottlerocket-bootstrap/pkg/kubeadm/testdata/manifests/kube-apiserver create mode 100644 projects/aws/bottlerocket-bootstrap/pkg/kubeadm/testdata/manifests/kube-vip diff --git a/projects/aws/bottlerocket-bootstrap/CHECKSUMS b/projects/aws/bottlerocket-bootstrap/CHECKSUMS index c155a5f0fd..dfcc8872a3 100644 --- a/projects/aws/bottlerocket-bootstrap/CHECKSUMS +++ b/projects/aws/bottlerocket-bootstrap/CHECKSUMS @@ -1,4 +1,4 @@ -04741f6569abd5ee0fbda4a47fb424c14e9930dbdc665238cc8e27ceb675eb53 _output/bin/bottlerocket-bootstrap/linux-amd64/bottlerocket-bootstrap -1fd36d9a1080f1ec489f0402fec9a7ab6c7cc68ae607babcb82ab55103c53558 _output/bin/bottlerocket-bootstrap/linux-amd64/bottlerocket-bootstrap-snow -688205c8ed160ba21a8f695d62a3fc98ad1ac344b7ff4df4ad95578286594bb4 _output/bin/bottlerocket-bootstrap/linux-arm64/bottlerocket-bootstrap -9b7d5023b692b488281f8b61aebb46fa7bf50338a4dc224624eac63661f8b65a _output/bin/bottlerocket-bootstrap/linux-arm64/bottlerocket-bootstrap-snow +e0092f5d1504bfeb2b1e52c6209c60a442345896e14e012dd5bc2cc20b9062a5 _output/bin/bottlerocket-bootstrap/linux-amd64/bottlerocket-bootstrap +df383df218dc2fbf5d73cd5e8308fad0a39a8e002f3c81b97dfba68276106e00 _output/bin/bottlerocket-bootstrap/linux-amd64/bottlerocket-bootstrap-snow +1d957f9a6daf7900b396f7d6c2ce9b68955c71369d880e0f8ba9e96be01843df _output/bin/bottlerocket-bootstrap/linux-arm64/bottlerocket-bootstrap +56dbda04ac02d58c1540a52be84a2f3a74f57761545cfe6e66e51ad2c65203a6 _output/bin/bottlerocket-bootstrap/linux-arm64/bottlerocket-bootstrap-snow diff --git a/projects/aws/bottlerocket-bootstrap/go.mod b/projects/aws/bottlerocket-bootstrap/go.mod index e000482b09..1078f56f1c 100644 --- a/projects/aws/bottlerocket-bootstrap/go.mod +++ b/projects/aws/bottlerocket-bootstrap/go.mod @@ -10,14 +10,14 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 github.com/godbus/dbus/v5 v5.1.0 github.com/golang/mock v1.6.0 - github.com/google/go-cmp v0.5.9 - github.com/onsi/gomega v1.27.6 + github.com/google/go-cmp v0.6.0 + github.com/onsi/gomega v1.29.0 github.com/pkg/errors v0.9.1 github.com/vishvananda/netlink v1.1.0 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.28.2 - k8s.io/apimachinery v0.28.2 - k8s.io/client-go v0.28.2 + k8s.io/api v0.29.2 + k8s.io/apimachinery v0.29.2 + k8s.io/client-go v0.29.2 sigs.k8s.io/yaml v1.3.0 ) @@ -31,29 +31,92 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.2 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.23.1 // indirect github.com/aws/smithy-go v1.15.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/coreos/go-semver v0.3.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-logr/logr v1.2.4 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/imdario/mergo v0.3.16 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_golang v1.16.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect - golang.org/x/net v0.17.0 // indirect + github.com/vishvananda/netns v0.0.4 // indirect + go.etcd.io/etcd/api/v3 v3.5.10 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect + go.etcd.io/etcd/client/v3 v3.5.10 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.19.0 // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.100.1 // indirect + k8s.io/cluster-bootstrap v0.0.0 // indirect + k8s.io/component-base v0.0.0 // indirect + k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect +) + +require k8s.io/kubernetes v1.29.2 + +replace ( + k8s.io/api => k8s.io/api v0.29.2 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.29.2 + k8s.io/apimachinery => k8s.io/apimachinery v0.29.2 + k8s.io/apiserver => k8s.io/apiserver v0.29.2 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.29.2 + k8s.io/client-go => k8s.io/client-go v0.29.2 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.29.2 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.29.2 + k8s.io/code-generator => k8s.io/code-generator v0.29.2 + k8s.io/component-base => k8s.io/component-base v0.29.2 + k8s.io/component-helpers => k8s.io/component-helpers v0.29.2 + k8s.io/controller-manager => k8s.io/controller-manager v0.29.2 + k8s.io/cri-api => k8s.io/cri-api v0.29.2 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.29.2 + k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.29.2 + k8s.io/kms => k8s.io/kms v0.29.2 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.29.2 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.29.2 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.29.2 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.29.2 + k8s.io/kubectl => k8s.io/kubectl v0.29.2 + k8s.io/kubelet => k8s.io/kubelet v0.29.2 + k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.29.2 + k8s.io/metrics => k8s.io/metrics v0.29.2 + k8s.io/mount-utils => k8s.io/mount-utils v0.29.2 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.29.2 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.2 ) diff --git a/projects/aws/bottlerocket-bootstrap/go.sum b/projects/aws/bottlerocket-bootstrap/go.sum index 8fb80481a6..d11f7a7e9d 100644 --- a/projects/aws/bottlerocket-bootstrap/go.sum +++ b/projects/aws/bottlerocket-bootstrap/go.sum @@ -24,18 +24,33 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.23.1 h1:ASNYk1ypWAxRhJjKS0jBnTUeDl7H github.com/aws/aws-sdk-go-v2/service/sts v1.23.1/go.mod h1:2cnsAhVT3mqusovc2stUSUrSBGTcX9nh8Tu6xh//2eI= github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= @@ -44,67 +59,116 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE= -github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/etcd/api/v3 v3.5.10 h1:szRajuUUbLyppkhs9K6BRtjY37l66XQQmw7oZRANE4k= +go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI= +go.etcd.io/etcd/client/pkg/v3 v3.5.10 h1:kfYIdQftBnbAq8pUWFXfpuuxFSKzlmM5cSn76JByiT0= +go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U= +go.etcd.io/etcd/client/v3 v3.5.10 h1:W9TXNZ+oB3MCd/8UjxHTWK5J9Nquw9fQBLJd5ne5/Ao= +go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -112,10 +176,11 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -131,60 +196,82 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= -k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= -k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= -k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= -k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= -k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= +k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= +k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= +k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= +k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= +k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= +k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= +k8s.io/cluster-bootstrap v0.29.2 h1:CJ8kNpm6vqPX6laBEPGoEFpVQ0XmzgXMdQosvd5m2OA= +k8s.io/cluster-bootstrap v0.29.2/go.mod h1:75qXUXImrhRHglBCQsBvZrS4uJFyaDinOWLWbbaRRH0= +k8s.io/component-base v0.29.2 h1:lpiLyuvPA9yV1aQwGLENYyK7n/8t6l3nn3zAtFTJYe8= +k8s.io/component-base v0.29.2/go.mod h1:BfB3SLrefbZXiBfbM+2H1dlat21Uewg/5qtKOl8degM= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/kubernetes v1.29.2 h1:8hh1cntqdulanjQt7wSSSsJfBgOyx6fUdFWslvGL5m0= +k8s.io/kubernetes v1.29.2/go.mod h1:xZPKU0yO0CBbLTnbd+XGyRmmtmaVuJykDb8gNCkeeUE= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/projects/aws/bottlerocket-bootstrap/pkg/constants/constants.go b/projects/aws/bottlerocket-bootstrap/pkg/constants/constants.go new file mode 100644 index 0000000000..947689b2ea --- /dev/null +++ b/projects/aws/bottlerocket-bootstrap/pkg/constants/constants.go @@ -0,0 +1,682 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package constants + +import ( + "fmt" + "net" + "os" + "path/filepath" + "strings" + "time" + + "github.com/pkg/errors" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/version" + apimachineryversion "k8s.io/apimachinery/pkg/version" + componentversion "k8s.io/component-base/version" + netutils "k8s.io/utils/net" +) + +const ( + // KubernetesDir is the directory Kubernetes owns for storing various configuration files + KubernetesDir = "/etc/kubernetes" + // ManifestsSubDirName defines directory name to store manifests + ManifestsSubDirName = "manifests" + // TempDirForKubeadm defines temporary directory for kubeadm + // should be joined with KubernetesDir. + TempDirForKubeadm = "tmp" + + // CertificateBackdate defines the offset applied to notBefore for CA certificates generated by kubeadm + CertificateBackdate = time.Minute * 5 + // CertificateValidity defines the validity for all the signed certificates generated by kubeadm + CertificateValidity = time.Hour * 24 * 365 + + // DefaultCertificateDir defines default certificate directory + DefaultCertificateDir = "pki" + + // CACertAndKeyBaseName defines certificate authority base name + CACertAndKeyBaseName = "ca" + // CACertName defines certificate name + CACertName = "ca.crt" + // CAKeyName defines certificate name + CAKeyName = "ca.key" + + // APIServerCertAndKeyBaseName defines API's server certificate and key base name + APIServerCertAndKeyBaseName = "apiserver" + // APIServerCertName defines API's server certificate name + APIServerCertName = "apiserver.crt" + // APIServerKeyName defines API's server key name + APIServerKeyName = "apiserver.key" + // APIServerCertCommonName defines API's server certificate common name (CN) + APIServerCertCommonName = "kube-apiserver" + + // APIServerKubeletClientCertAndKeyBaseName defines kubelet client certificate and key base name + APIServerKubeletClientCertAndKeyBaseName = "apiserver-kubelet-client" + // APIServerKubeletClientCertName defines kubelet client certificate name + APIServerKubeletClientCertName = "apiserver-kubelet-client.crt" + // APIServerKubeletClientKeyName defines kubelet client key name + APIServerKubeletClientKeyName = "apiserver-kubelet-client.key" + // APIServerKubeletClientCertCommonName defines kubelet client certificate common name (CN) + APIServerKubeletClientCertCommonName = "kube-apiserver-kubelet-client" + + // EtcdCACertAndKeyBaseName defines etcd's CA certificate and key base name + EtcdCACertAndKeyBaseName = "etcd/ca" + // EtcdCACertName defines etcd's CA certificate name + EtcdCACertName = "etcd/ca.crt" + // EtcdCAKeyName defines etcd's CA key name + EtcdCAKeyName = "etcd/ca.key" + + // EtcdServerCertAndKeyBaseName defines etcd's server certificate and key base name + EtcdServerCertAndKeyBaseName = "etcd/server" + // EtcdServerCertName defines etcd's server certificate name + EtcdServerCertName = "etcd/server.crt" + // EtcdServerKeyName defines etcd's server key name + EtcdServerKeyName = "etcd/server.key" + + // EtcdListenClientPort defines the port etcd listen on for client traffic + EtcdListenClientPort = 2379 + // EtcdMetricsPort is the port at which to obtain etcd metrics and health status + EtcdMetricsPort = 2381 + + // EtcdPeerCertAndKeyBaseName defines etcd's peer certificate and key base name + EtcdPeerCertAndKeyBaseName = "etcd/peer" + // EtcdPeerCertName defines etcd's peer certificate name + EtcdPeerCertName = "etcd/peer.crt" + // EtcdPeerKeyName defines etcd's peer key name + EtcdPeerKeyName = "etcd/peer.key" + + // EtcdListenPeerPort defines the port etcd listen on for peer traffic + EtcdListenPeerPort = 2380 + + // EtcdHealthcheckClientCertAndKeyBaseName defines etcd's healthcheck client certificate and key base name + EtcdHealthcheckClientCertAndKeyBaseName = "etcd/healthcheck-client" + // EtcdHealthcheckClientCertName defines etcd's healthcheck client certificate name + EtcdHealthcheckClientCertName = "etcd/healthcheck-client.crt" + // EtcdHealthcheckClientKeyName defines etcd's healthcheck client key name + EtcdHealthcheckClientKeyName = "etcd/healthcheck-client.key" + // EtcdHealthcheckClientCertCommonName defines etcd's healthcheck client certificate common name (CN) + EtcdHealthcheckClientCertCommonName = "kube-etcd-healthcheck-client" + + // APIServerEtcdClientCertAndKeyBaseName defines apiserver's etcd client certificate and key base name + APIServerEtcdClientCertAndKeyBaseName = "apiserver-etcd-client" + // APIServerEtcdClientCertName defines apiserver's etcd client certificate name + APIServerEtcdClientCertName = "apiserver-etcd-client.crt" + // APIServerEtcdClientKeyName defines apiserver's etcd client key name + APIServerEtcdClientKeyName = "apiserver-etcd-client.key" + // APIServerEtcdClientCertCommonName defines apiserver's etcd client certificate common name (CN) + APIServerEtcdClientCertCommonName = "kube-apiserver-etcd-client" + + // ServiceAccountKeyBaseName defines SA key base name + ServiceAccountKeyBaseName = "sa" + // ServiceAccountPublicKeyName defines SA public key base name + ServiceAccountPublicKeyName = "sa.pub" + // ServiceAccountPrivateKeyName defines SA private key base name + ServiceAccountPrivateKeyName = "sa.key" + + // FrontProxyCACertAndKeyBaseName defines front proxy CA certificate and key base name + FrontProxyCACertAndKeyBaseName = "front-proxy-ca" + // FrontProxyCACertName defines front proxy CA certificate name + FrontProxyCACertName = "front-proxy-ca.crt" + // FrontProxyCAKeyName defines front proxy CA key name + FrontProxyCAKeyName = "front-proxy-ca.key" + + // FrontProxyClientCertAndKeyBaseName defines front proxy certificate and key base name + FrontProxyClientCertAndKeyBaseName = "front-proxy-client" + // FrontProxyClientCertName defines front proxy certificate name + FrontProxyClientCertName = "front-proxy-client.crt" + // FrontProxyClientKeyName defines front proxy key name + FrontProxyClientKeyName = "front-proxy-client.key" + // FrontProxyClientCertCommonName defines front proxy certificate common name + FrontProxyClientCertCommonName = "front-proxy-client" //used as subject.commonname attribute (CN) + + // AdminKubeConfigFileName defines name for the kubeconfig aimed to be used by the admin of the cluster + AdminKubeConfigFileName = "admin.conf" + // SuperAdminKubeConfigFileName defines name for the kubeconfig aimed to be used by the super-admin of the cluster + SuperAdminKubeConfigFileName = "super-admin.conf" + + // KubeletBootstrapKubeConfigFileName defines the file name for the kubeconfig that the kubelet will use to do + // the TLS bootstrap to get itself an unique credential + KubeletBootstrapKubeConfigFileName = "bootstrap-kubelet.conf" + + // KubeletKubeConfigFileName defines the file name for the kubeconfig that the control-plane kubelet will use for talking + // to the API server + KubeletKubeConfigFileName = "kubelet.conf" + // ControllerManagerKubeConfigFileName defines the file name for the controller manager's kubeconfig file + ControllerManagerKubeConfigFileName = "controller-manager.conf" + // SchedulerKubeConfigFileName defines the file name for the scheduler's kubeconfig file + SchedulerKubeConfigFileName = "scheduler.conf" + + // Some well-known users, groups, roles and clusterrolebindings in the core Kubernetes authorization system + + // ControllerManagerUser defines the well-known user the controller-manager should be authenticated as + ControllerManagerUser = "system:kube-controller-manager" + // SchedulerUser defines the well-known user the scheduler should be authenticated as + SchedulerUser = "system:kube-scheduler" + // NodesUserPrefix defines the user name prefix as requested by the Node authorizer. + NodesUserPrefix = "system:node:" + // SystemPrivilegedGroup defines the well-known group for the apiservers. This group is also superuser by default + // (i.e. bound to the cluster-admin ClusterRole) + SystemPrivilegedGroup = "system:masters" + // NodesGroup defines the well-known group for all nodes. + NodesGroup = "system:nodes" + // NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in + NodeBootstrapTokenAuthGroup = "system:bootstrappers:kubeadm:default-node-token" + // KubeProxyClusterRoleName sets the name for the kube-proxy ClusterRole + KubeProxyClusterRoleName = "system:node-proxier" + // NodeBootstrapperClusterRoleName defines the name of the auto-bootstrapped ClusterRole for letting someone post a CSR + NodeBootstrapperClusterRoleName = "system:node-bootstrapper" + // CSRAutoApprovalClusterRoleName defines the name of the auto-bootstrapped ClusterRole for making the csrapprover controller auto-approve the CSR + // Starting from v1.8, CSRAutoApprovalClusterRoleName is automatically created by the API server on startup + CSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:nodeclient" + // NodeSelfCSRAutoApprovalClusterRoleName is a role defined in default 1.8 RBAC policies for automatic CSR approvals for automatically rotated node certificates + NodeSelfCSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:selfnodeclient" + // NodesClusterRoleBinding defines the well-known ClusterRoleBinding which binds the too permissive system:node + // ClusterRole to the system:nodes group. Since kubeadm is using the Node Authorizer, this ClusterRoleBinding's + // system:nodes group subject is removed if present. + NodesClusterRoleBinding = "system:node" + + // KubeletBaseConfigMapRole defines the base kubelet configuration ConfigMap. + KubeletBaseConfigMapRole = "kubeadm:kubelet-config" + // KubeProxyClusterRoleBindingName sets the name for the kube-proxy CluterRoleBinding + KubeProxyClusterRoleBindingName = "kubeadm:node-proxier" + // NodeKubeletBootstrap defines the name of the ClusterRoleBinding that lets kubelets post CSRs + NodeKubeletBootstrap = "kubeadm:kubelet-bootstrap" + // GetNodesClusterRoleName defines the name of the ClusterRole and ClusterRoleBinding to get nodes + GetNodesClusterRoleName = "kubeadm:get-nodes" + // NodeAutoApproveBootstrapClusterRoleBinding defines the name of the ClusterRoleBinding that makes the csrapprover approve node CSRs + NodeAutoApproveBootstrapClusterRoleBinding = "kubeadm:node-autoapprove-bootstrap" + // NodeAutoApproveCertificateRotationClusterRoleBinding defines name of the ClusterRoleBinding that makes the csrapprover approve node auto rotated CSRs + NodeAutoApproveCertificateRotationClusterRoleBinding = "kubeadm:node-autoapprove-certificate-rotation" + // ClusterAdminsGroupAndClusterRoleBinding is the name of the Group used for kubeadm generated cluster + // admin credentials and the name of the ClusterRoleBinding that binds the same Group to the "cluster-admin" + // built-in ClusterRole. + ClusterAdminsGroupAndClusterRoleBinding = "kubeadm:cluster-admins" + + // KubernetesAPICallTimeout specifies how long kubeadm should wait for API calls + KubernetesAPICallTimeout = 1 * time.Minute + // KubernetesAPICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation + KubernetesAPICallRetryInterval = 500 * time.Millisecond + + // DiscoveryTimeout specifies the default discovery timeout for kubeadm (used unless one is specified in the JoinConfiguration) + DiscoveryTimeout = 5 * time.Minute + // DiscoveryRetryInterval specifies how long kubeadm should wait before retrying to connect to the control-plane when doing discovery + DiscoveryRetryInterval = 5 * time.Second + + // TLSBootstrapTimeout specifies how long kubeadm should wait for the kubelet to perform the TLS Bootstrap + TLSBootstrapTimeout = 5 * time.Minute + // TLSBootstrapRetryInterval specifies how long kubeadm should wait before retrying the TLS Bootstrap check + TLSBootstrapRetryInterval = 1 * time.Second + + // StaticPodMirroringTimeout specifies how much time kubeadm should wait for the static pods + // to be mirrored on the API server. + StaticPodMirroringTimeout = 30 * time.Second + // StaticPodMirroringRetryInterval specifies how often to check if static pods are mirrored at the + // API server. + StaticPodMirroringRetryInterval = 500 * time.Millisecond + + // EtcdAPICallTimeout specifies how much time to wait for completion of requests against the etcd API. + EtcdAPICallTimeout = 2 * time.Minute + // EtcdAPICallRetryInterval specifies how frequently to retry requests against the etcd API. + EtcdAPICallRetryInterval = 500 * time.Millisecond + + // ControlPlaneComponentHealthCheckTimeout specifies the default control plane component health check timeout + ControlPlaneComponentHealthCheckTimeout = 4 * time.Minute + + // KubeletHealthCheckTimeout specifies the default kubelet timeout + KubeletHealthCheckTimeout = 4 * time.Minute + + // PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed + PullImageRetry = 5 + // RemoveContainerRetry specifies how many times ContainerRuntime retries when removing container failed + RemoveContainerRetry = 5 + + // MinimumAddressesInServiceSubnet defines minimum amount of nodes the Service subnet should allow. + // We need at least ten, because the DNS service is always at the tenth cluster clusterIP + MinimumAddressesInServiceSubnet = 10 + + // MaximumBitsForServiceSubnet defines maximum possible size of the service subnet in terms of bits. + // For example, if the value is 20, then the largest supported service subnet is /12 for IPv4 and /108 for IPv6. + // Note however that anything in between /108 and /112 will be clamped to /112 due to the limitations of the underlying allocation logic. + // TODO: https://github.com/kubernetes/enhancements/pull/1881 + MaximumBitsForServiceSubnet = 20 + + // MinimumAddressesInPodSubnet defines minimum amount of pods in the cluster. + // We need at least more than services, an IPv4 /28 or IPv6 /128 subnet means 14 util addresses + MinimumAddressesInPodSubnet = 14 + + // PodSubnetNodeMaskMaxDiff is limited to 16 due to an issue with uncompressed IP bitmap in core: + // xref: #44918 + // The node subnet mask size must be no more than the pod subnet mask size + 16 + PodSubnetNodeMaskMaxDiff = 16 + + // DefaultCertTokenDuration specifies the default amount of time that the token used by upload certs will be valid + // Default behaviour is 2 hours + DefaultCertTokenDuration = 2 * time.Hour + + // CertificateKeySize specifies the size of the key used to encrypt certificates on uploadcerts phase + CertificateKeySize = 32 + + // LabelNodeRoleControlPlane specifies that a node hosts control-plane components + LabelNodeRoleControlPlane = "node-role.kubernetes.io/control-plane" + + // LabelExcludeFromExternalLB can be set on a node to exclude it from external load balancers. + // This is added to control plane nodes to preserve backwards compatibility with a legacy behavior. + LabelExcludeFromExternalLB = "node.kubernetes.io/exclude-from-external-load-balancers" + + // AnnotationKubeadmCRISocket specifies the annotation kubeadm uses to preserve the crisocket information given to kubeadm at + // init/join time for use later. kubeadm annotates the node object with this information + AnnotationKubeadmCRISocket = "kubeadm.alpha.kubernetes.io/cri-socket" + + // KubeadmConfigConfigMap specifies in what ConfigMap in the kube-system namespace the `kubeadm init` configuration should be stored + KubeadmConfigConfigMap = "kubeadm-config" + + // ClusterConfigurationConfigMapKey specifies in what ConfigMap key the cluster configuration should be stored + ClusterConfigurationConfigMapKey = "ClusterConfiguration" + + // KubeProxyConfigMap specifies in what ConfigMap in the kube-system namespace the kube-proxy configuration should be stored + KubeProxyConfigMap = "kube-proxy" + + // KubeProxyConfigMapKey specifies in what ConfigMap key the component config of kube-proxy should be stored + KubeProxyConfigMapKey = "config.conf" + + // KubeletBaseConfigurationConfigMap specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored + KubeletBaseConfigurationConfigMap = "kubelet-config" + + // KubeletBaseConfigurationConfigMapKey specifies in what ConfigMap key the initial remote configuration of kubelet should be stored + KubeletBaseConfigurationConfigMapKey = "kubelet" + + // KubeletRunDirectory specifies the directory where the kubelet runtime information is stored. + KubeletRunDirectory = "/var/lib/kubelet" + + // KubeletConfigurationFileName specifies the file name on the node which stores initial remote configuration of kubelet + // This file should exist under KubeletRunDirectory + KubeletConfigurationFileName = "config.yaml" + + // KubeletEnvFileName is a file "kubeadm init" writes at runtime. Using that interface, kubeadm can customize certain + // kubelet flags conditionally based on the environment at runtime. Also, parameters given to the configuration file + // might be passed through this file. "kubeadm init" writes one variable, with the name ${KubeletEnvFileVariableName}. + // This file should exist under KubeletRunDirectory + KubeletEnvFileName = "kubeadm-flags.env" + + // KubeletEnvFileVariableName specifies the shell script variable name "kubeadm init" should write a value to in KubeletEnvFile + KubeletEnvFileVariableName = "KUBELET_KUBEADM_ARGS" + + // KubeletHealthzPort is the port of the kubelet healthz endpoint + KubeletHealthzPort = 10248 + + // MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports + MinExternalEtcdVersion = "3.4.13-4" + + // DefaultEtcdVersion indicates the default etcd version that kubeadm uses + DefaultEtcdVersion = "3.5.12-0" + + // Etcd defines variable used internally when referring to etcd component + Etcd = "etcd" + // KubeAPIServer defines variable used internally when referring to kube-apiserver component + KubeAPIServer = "kube-apiserver" + // KubeControllerManager defines variable used internally when referring to kube-controller-manager component + KubeControllerManager = "kube-controller-manager" + // KubeScheduler defines variable used internally when referring to kube-scheduler component + KubeScheduler = "kube-scheduler" + // KubeProxy defines variable used internally when referring to kube-proxy component + KubeProxy = "kube-proxy" + // CoreDNS defines variable used internally when referring to the CoreDNS component + CoreDNS = "CoreDNS" + // Kubelet defines variable used internally when referring to the Kubelet + Kubelet = "kubelet" + + // KubeCertificatesVolumeName specifies the name for the Volume that is used for injecting certificates to control plane components (can be both a hostPath volume or a projected, all-in-one volume) + KubeCertificatesVolumeName = "k8s-certs" + + // KubeConfigVolumeName specifies the name for the Volume that is used for injecting the kubeconfig to talk securely to the api server for a control plane component if applicable + KubeConfigVolumeName = "kubeconfig" + + // DefaultCIImageRepository points to image registry where CI uploads images from ci build job + DefaultCIImageRepository = "gcr.io/k8s-staging-ci-images" + + // CoreDNSConfigMap specifies in what ConfigMap in the kube-system namespace the CoreDNS config should be stored + CoreDNSConfigMap = "coredns" + + // CoreDNSDeploymentName specifies the name of the Deployment for CoreDNS add-on + CoreDNSDeploymentName = "coredns" + + // CoreDNSImageName specifies the name of the image for CoreDNS add-on + CoreDNSImageName = "coredns" + + // CoreDNSVersion is the version of CoreDNS to be deployed if it is used + CoreDNSVersion = "v1.11.1" + + // ClusterConfigurationKind is the string kind value for the ClusterConfiguration struct + ClusterConfigurationKind = "ClusterConfiguration" + + // InitConfigurationKind is the string kind value for the InitConfiguration struct + InitConfigurationKind = "InitConfiguration" + + // JoinConfigurationKind is the string kind value for the JoinConfiguration struct + JoinConfigurationKind = "JoinConfiguration" + + // ResetConfigurationKind is the string kind value for the ResetConfiguration struct + ResetConfigurationKind = "ResetConfiguration" + + // YAMLDocumentSeparator is the separator for YAML documents + // TODO: Find a better place for this constant + YAMLDocumentSeparator = "---\n" + + // CIKubernetesVersionPrefix is the prefix for CI Kubernetes version + CIKubernetesVersionPrefix = "ci/" + + // DefaultAPIServerBindAddress is the default bind address for the API Server + DefaultAPIServerBindAddress = "0.0.0.0" + + // ControlPlaneNumCPU is the number of CPUs required on control-plane + ControlPlaneNumCPU = 2 + + // ControlPlaneMem is the number of megabytes of memory required on the control-plane + // Below that amount of RAM running a stable control plane would be difficult. + ControlPlaneMem = 1700 + + // KubeadmCertsSecret specifies in what Secret in the kube-system namespace the certificates should be stored + KubeadmCertsSecret = "kubeadm-certs" + + // KubeletPort is the default port for the kubelet server on each host machine. + // May be overridden by a flag at startup. + KubeletPort = 10250 + // KubeSchedulerPort is the default port for the scheduler status server. + // May be overridden by a flag at startup. + KubeSchedulerPort = 10259 + // KubeControllerManagerPort is the default port for the controller manager status server. + // May be overridden by a flag at startup. + KubeControllerManagerPort = 10257 + + // EtcdAdvertiseClientUrlsAnnotationKey is the annotation key on every etcd pod, describing the + // advertise client URLs + EtcdAdvertiseClientUrlsAnnotationKey = "kubeadm.kubernetes.io/etcd.advertise-client-urls" + // KubeAPIServerAdvertiseAddressEndpointAnnotationKey is the annotation key on every apiserver pod, + // describing the API endpoint (advertise address and bind port of the api server) + KubeAPIServerAdvertiseAddressEndpointAnnotationKey = "kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint" + // ComponentConfigHashAnnotationKey holds the config map annotation key that kubeadm uses to store + // a SHA256 sum to check for user changes + ComponentConfigHashAnnotationKey = "kubeadm.kubernetes.io/component-config.hash" + + // ControlPlaneTier is the value used in the tier label to identify control plane components + ControlPlaneTier = "control-plane" + + // Mode* constants were copied from pkg/kubeapiserver/authorizer/modes + // to avoid kubeadm dependency on the internal module + // TODO: share Mode* constants in component config + + // ModeAlwaysAllow is the mode to set all requests as authorized + ModeAlwaysAllow string = "AlwaysAllow" + // ModeAlwaysDeny is the mode to set no requests as authorized + ModeAlwaysDeny string = "AlwaysDeny" + // ModeABAC is the mode to use Attribute Based Access Control to authorize + ModeABAC string = "ABAC" + // ModeWebhook is the mode to make an external webhook call to authorize + ModeWebhook string = "Webhook" + // ModeRBAC is the mode to use Role Based Access Control to authorize + ModeRBAC string = "RBAC" + // ModeNode is an authorization mode that authorizes API requests made by kubelets. + ModeNode string = "Node" + + // PauseVersion indicates the default pause image version for kubeadm + PauseVersion = "3.9" + + // CgroupDriverSystemd holds the systemd driver type + CgroupDriverSystemd = "systemd" + + // KubeControllerManagerUserName is the username of the user that kube-controller-manager runs as. + KubeControllerManagerUserName string = "kubeadm-kcm" + // KubeAPIServerUserName is the username of the user that kube-apiserver runs as. + KubeAPIServerUserName string = "kubeadm-kas" + // KubeSchedulerUserName is the username of the user that kube-scheduler runs as. + KubeSchedulerUserName string = "kubeadm-ks" + // EtcdUserName is the username of the user that etcd runs as. + EtcdUserName string = "kubeadm-etcd" + // ServiceAccountKeyReadersGroupName is the group of users that are allowed to read the service account private key. + ServiceAccountKeyReadersGroupName string = "kubeadm-sa-key-readers" +) + +var ( + // ControlPlaneTaint is the taint to apply on the PodSpec for being able to run that Pod on the control-plane + ControlPlaneTaint = v1.Taint{ + Key: LabelNodeRoleControlPlane, + Effect: v1.TaintEffectNoSchedule, + } + + // ControlPlaneToleration is the toleration to apply on the PodSpec for being able to run that Pod on the control-plane + ControlPlaneToleration = v1.Toleration{ + Key: LabelNodeRoleControlPlane, + Effect: v1.TaintEffectNoSchedule, + } + + // ControlPlaneComponents defines the control-plane component names + ControlPlaneComponents = []string{KubeAPIServer, KubeControllerManager, KubeScheduler} + + // MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy + MinimumControlPlaneVersion = getSkewedKubernetesVersion(-1) + + // MinimumKubeletVersion specifies the minimum version of kubelet which kubeadm supports + MinimumKubeletVersion = getSkewedKubernetesVersion(-3) + + // CurrentKubernetesVersion specifies current Kubernetes version supported by kubeadm + CurrentKubernetesVersion = getSkewedKubernetesVersion(0) + + // SupportedEtcdVersion lists officially supported etcd versions with corresponding Kubernetes releases + SupportedEtcdVersion = map[uint8]string{ + 22: "3.5.12-0", + 23: "3.5.12-0", + 24: "3.5.12-0", + 25: "3.5.12-0", + 26: "3.5.12-0", + 27: "3.5.12-0", + 28: "3.5.12-0", + 29: "3.5.12-0", + 30: "3.5.12-0", + } + + // KubeadmCertsClusterRoleName sets the name for the ClusterRole that allows + // the bootstrap tokens to access the kubeadm-certs Secret during the join of a new control-plane + KubeadmCertsClusterRoleName = fmt.Sprintf("kubeadm:%s", KubeadmCertsSecret) + + // defaultKubernetesPlaceholderVersion is a placeholder version in case the component-base + // version was not populated during build. + defaultKubernetesPlaceholderVersion = version.MustParseSemantic("v1.0.0-placeholder-version") +) + +// getSkewedKubernetesVersion returns the current MAJOR.(MINOR+n).0 Kubernetes version with a skew of 'n' +// It uses the kubeadm version provided by the 'component-base/version' package. This version must be populated +// by passing linker flags during the kubeadm build process. If the version is empty, assume that kubeadm +// was either build incorrectly or this code is running in unit tests. +func getSkewedKubernetesVersion(n int) *version.Version { + versionInfo := componentversion.Get() + return getSkewedKubernetesVersionImpl(&versionInfo, n) +} + +func getSkewedKubernetesVersionImpl(versionInfo *apimachineryversion.Info, n int) *version.Version { + // TODO: update if the kubeadm version gets decoupled from the Kubernetes version. + // This would require keeping track of the supported skew in a table. + // More changes would be required if the kubelet version one day decouples from that of Kubernetes. + var ver *version.Version + if len(versionInfo.Major) == 0 { + return defaultKubernetesPlaceholderVersion + } + ver = version.MustParseSemantic(versionInfo.GitVersion) + // Append the MINOR version skew. + // TODO: handle the case of Kubernetes moving to v2.0 or having MAJOR version updates in the future. + // This would require keeping track (in a table) of the last MINOR for a particular MAJOR. + minor := uint(int(ver.Minor()) + n) + return version.MustParseSemantic(fmt.Sprintf("v%d.%d.0", ver.Major(), minor)) +} + +// EtcdSupportedVersion returns officially supported version of etcd for a specific Kubernetes release +// If passed version is not in the given list, the function returns the nearest version with a warning +func EtcdSupportedVersion(supportedEtcdVersion map[uint8]string, versionString string) (etcdVersion *version.Version, warning, err error) { + kubernetesVersion, err := version.ParseSemantic(versionString) + if err != nil { + return nil, nil, err + } + desiredVersion, etcdStringVersion := uint8(kubernetesVersion.Minor()), "" + + min, max := ^uint8(0), uint8(0) + for k, v := range supportedEtcdVersion { + if desiredVersion == k { + etcdStringVersion = v + break + } + if k < min { + min = k + } + if k > max { + max = k + } + } + + if len(etcdStringVersion) == 0 { + if desiredVersion < min { + etcdStringVersion = supportedEtcdVersion[min] + } + if desiredVersion > max { + etcdStringVersion = supportedEtcdVersion[max] + } + warning = errors.Errorf("could not find officially supported version of etcd for Kubernetes %s, falling back to the nearest etcd version (%s)", + versionString, etcdStringVersion) + } + + etcdVersion, err = version.ParseSemantic(etcdStringVersion) + if err != nil { + return nil, nil, err + } + + return etcdVersion, warning, nil +} + +// GetStaticPodDirectory returns the location on the disk where the Static Pod should be present +func GetStaticPodDirectory() string { + return filepath.Join(KubernetesDir, ManifestsSubDirName) +} + +// GetStaticPodFilepath returns the location on the disk where the Static Pod should be present +func GetStaticPodFilepath(componentName, manifestsDir string) string { + return filepath.Join(manifestsDir, componentName+".yaml") +} + +// GetAdminKubeConfigPath returns the location on the disk where admin kubeconfig is located by default +func GetAdminKubeConfigPath() string { + return filepath.Join(KubernetesDir, AdminKubeConfigFileName) +} + +// GetSuperAdminKubeConfigPath returns the location on the disk where admin kubeconfig is located by default +func GetSuperAdminKubeConfigPath() string { + return filepath.Join(KubernetesDir, SuperAdminKubeConfigFileName) +} + +// GetBootstrapKubeletKubeConfigPath returns the location on the disk where bootstrap kubelet kubeconfig is located by default +func GetBootstrapKubeletKubeConfigPath() string { + return filepath.Join(KubernetesDir, KubeletBootstrapKubeConfigFileName) +} + +// GetKubeletKubeConfigPath returns the location on the disk where kubelet kubeconfig is located by default +func GetKubeletKubeConfigPath() string { + return filepath.Join(KubernetesDir, KubeletKubeConfigFileName) +} + +// CreateTempDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp (not using /tmp as that would potentially be dangerous) +func CreateTempDirForKubeadm(kubernetesDir, dirName string) (string, error) { + tempDir := filepath.Join(KubernetesDir, TempDirForKubeadm) + if len(kubernetesDir) != 0 { + tempDir = filepath.Join(kubernetesDir, TempDirForKubeadm) + } + + // creates target folder if not already exists + if err := os.MkdirAll(tempDir, 0700); err != nil { + return "", errors.Wrapf(err, "failed to create directory %q", tempDir) + } + + tempDir, err := os.MkdirTemp(tempDir, dirName) + if err != nil { + return "", errors.Wrap(err, "couldn't create a temporary directory") + } + return tempDir, nil +} + +// CreateTimestampDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp formatted with the current date +func CreateTimestampDirForKubeadm(kubernetesDir, dirName string) (string, error) { + tempDir := filepath.Join(KubernetesDir, TempDirForKubeadm) + if len(kubernetesDir) != 0 { + tempDir = filepath.Join(kubernetesDir, TempDirForKubeadm) + } + + // creates target folder if not already exists + if err := os.MkdirAll(tempDir, 0700); err != nil { + return "", errors.Wrapf(err, "failed to create directory %q", tempDir) + } + + timestampDirName := fmt.Sprintf("%s-%s", dirName, time.Now().Format("2006-01-02-15-04-05")) + timestampDir := filepath.Join(tempDir, timestampDirName) + if err := os.Mkdir(timestampDir, 0700); err != nil { + return "", errors.Wrap(err, "could not create timestamp directory") + } + + return timestampDir, nil +} + +// GetDNSIP returns a dnsIP, which is 10th IP in svcSubnet CIDR range +func GetDNSIP(svcSubnetList string) (net.IP, error) { + // Get the service subnet CIDR + svcSubnetCIDR, err := GetKubernetesServiceCIDR(svcSubnetList) + if err != nil { + return nil, errors.Wrapf(err, "unable to get internal Kubernetes Service IP from the given service CIDR (%s)", svcSubnetList) + } + + // Selects the 10th IP in service subnet CIDR range as dnsIP + dnsIP, err := netutils.GetIndexedIP(svcSubnetCIDR, 10) + if err != nil { + return nil, errors.Wrap(err, "unable to get internal Kubernetes Service IP from the given service CIDR") + } + + return dnsIP, nil +} + +// GetKubernetesServiceCIDR returns the default Service CIDR for the Kubernetes internal service +func GetKubernetesServiceCIDR(svcSubnetList string) (*net.IPNet, error) { + // The default service address family for the cluster is the address family of the first + // service cluster IP range configured via the `--service-cluster-ip-range` flag + // of the kube-controller-manager and kube-apiserver. + svcSubnets, err := netutils.ParseCIDRs(strings.Split(svcSubnetList, ",")) + if err != nil { + return nil, errors.Wrapf(err, "unable to parse ServiceSubnet %v", svcSubnetList) + } + if len(svcSubnets) == 0 { + return nil, errors.New("received empty ServiceSubnet") + } + return svcSubnets[0], nil +} + +// GetAPIServerVirtualIP returns the IP of the internal Kubernetes API service +func GetAPIServerVirtualIP(svcSubnetList string) (net.IP, error) { + svcSubnet, err := GetKubernetesServiceCIDR(svcSubnetList) + if err != nil { + return nil, errors.Wrap(err, "unable to get internal Kubernetes Service IP from the given service CIDR") + } + internalAPIServerVirtualIP, err := netutils.GetIndexedIP(svcSubnet, 1) + if err != nil { + return nil, errors.Wrapf(err, "unable to get the first IP address from the given CIDR: %s", svcSubnet.String()) + } + return internalAPIServerVirtualIP, nil +} diff --git a/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/controlplane_join.go b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/controlplane_join.go index af2c1479fd..cc95e736f0 100644 --- a/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/controlplane_join.go +++ b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/controlplane_join.go @@ -91,6 +91,8 @@ func controlPlaneJoin() error { return errors.Wrap(err, "Error enabling static pods") } + + // Now that etcd is up and running, check for other pod liveness err = utils.WaitForPods(podDefinitions) if err != nil { diff --git a/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/etcd/etcd.go b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/etcd/etcd.go new file mode 100644 index 0000000000..073d0f57ea --- /dev/null +++ b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/etcd/etcd.go @@ -0,0 +1,640 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcd + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/url" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" + "google.golang.org/grpc" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "github.com/eks-anywhere-build-tooling/aws/bottlerocket-bootstrap/pkg/constants" +) + +const etcdTimeout = 2 * time.Second + +// ErrNoMemberIDForPeerURL is returned when it is not possible to obtain a member ID +// from a given peer URL +var ErrNoMemberIDForPeerURL = errors.New("no member id found for peer URL") + +// ClusterInterrogator is an interface to get etcd cluster related information +type ClusterInterrogator interface { + CheckClusterHealth() error + WaitForClusterAvailable(retries int, retryInterval time.Duration) (bool, error) + Sync() error + ListMembers() ([]Member, error) + AddMember(name string, peerAddrs string) ([]Member, error) + AddMemberAsLearner(name string, peerAddrs string) ([]Member, error) + MemberPromote(learnerID uint64) error + GetMemberID(peerURL string) (uint64, error) + RemoveMember(id uint64) ([]Member, error) +} + +type etcdClient interface { + // Close shuts down the client's etcd connections. + Close() error + + // Endpoints lists the registered endpoints for the client. + Endpoints() []string + + // MemberList lists the current cluster membership. + MemberList(ctx context.Context) (*clientv3.MemberListResponse, error) + + // MemberAdd adds a new member into the cluster. + MemberAdd(ctx context.Context, peerAddrs []string) (*clientv3.MemberAddResponse, error) + + // MemberAddAsLearner adds a new learner member into the cluster. + MemberAddAsLearner(ctx context.Context, peerAddrs []string) (*clientv3.MemberAddResponse, error) + + // MemberRemove removes an existing member from the cluster. + MemberRemove(ctx context.Context, id uint64) (*clientv3.MemberRemoveResponse, error) + + // MemberPromote promotes a member from raft learner (non-voting) to raft voting member. + MemberPromote(ctx context.Context, id uint64) (*clientv3.MemberPromoteResponse, error) + + // Status gets the status of the endpoint. + Status(ctx context.Context, endpoint string) (*clientv3.StatusResponse, error) + + // Sync synchronizes client's endpoints with the known endpoints from the etcd membership. + Sync(ctx context.Context) error +} + +// Client provides connection parameters for an etcd cluster +type Client struct { + Endpoints []string + + newEtcdClient func(endpoints []string) (etcdClient, error) + + listMembersFunc func(timeout time.Duration) (*clientv3.MemberListResponse, error) +} + +// New creates a new EtcdCluster client +func New(endpoints []string, ca, cert, key string) (*Client, error) { + client := Client{Endpoints: endpoints} + + var err error + var tlsConfig *tls.Config + if ca != "" || cert != "" || key != "" { + tlsInfo := transport.TLSInfo{ + CertFile: cert, + KeyFile: key, + TrustedCAFile: ca, + } + tlsConfig, err = tlsInfo.ClientConfig() + if err != nil { + return nil, err + } + } + + client.newEtcdClient = func(endpoints []string) (etcdClient, error) { + return clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: etcdTimeout, + DialOptions: []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + }, + TLS: tlsConfig, + }) + } + + client.listMembersFunc = client.listMembers + + return &client, nil +} + +// NewFromCluster creates an etcd client for the etcd endpoints present in etcd member list. In order to compose this information, +// it will first discover at least one etcd endpoint to connect to. Once created, the client synchronizes client's endpoints with +// the known endpoints from the etcd membership API, since it is the authoritative source of truth for the list of available members. +func NewFromCluster(client clientset.Interface, certificatesDir string) (*Client, error) { + // Discover at least one etcd endpoint to connect to by inspecting the existing etcd pods + + // Get the list of etcd endpoints + endpoints, err := getEtcdEndpoints(client) + if err != nil { + return nil, err + } + klog.V(1).Infof("etcd endpoints read from pods: %s", strings.Join(endpoints, ",")) + + // Creates an etcd client + etcdClient, err := New( + endpoints, + filepath.Join(certificatesDir, constants.EtcdCACertName), + filepath.Join(certificatesDir, constants.EtcdHealthcheckClientCertName), + filepath.Join(certificatesDir, constants.EtcdHealthcheckClientKeyName), + ) + if err != nil { + return nil, errors.Wrapf(err, "error creating etcd client for %v endpoints", endpoints) + } + + // synchronizes client's endpoints with the known endpoints from the etcd membership. + err = etcdClient.Sync() + if err != nil { + return nil, errors.Wrap(err, "error syncing endpoints with etcd") + } + klog.V(1).Infof("update etcd endpoints: %s", strings.Join(etcdClient.Endpoints, ",")) + + return etcdClient, nil +} + +// getEtcdEndpoints returns the list of etcd endpoints. +func getEtcdEndpoints(client clientset.Interface) ([]string, error) { + return getEtcdEndpointsWithRetry(client, + constants.StaticPodMirroringRetryInterval, constants.StaticPodMirroringTimeout) +} + +func getEtcdEndpointsWithRetry(client clientset.Interface, interval, timeout time.Duration) ([]string, error) { + return getRawEtcdEndpointsFromPodAnnotation(client, interval, timeout) +} + +// getRawEtcdEndpointsFromPodAnnotation returns the list of endpoints as reported on etcd's pod annotations using the given backoff +func getRawEtcdEndpointsFromPodAnnotation(client clientset.Interface, interval, timeout time.Duration) ([]string, error) { + etcdEndpoints := []string{} + var lastErr error + // Let's tolerate some unexpected transient failures from the API server or load balancers. Also, if + // static pods were not yet mirrored into the API server we want to wait for this propagation. + err := wait.PollUntilContextTimeout(context.Background(), interval, timeout, true, + func(_ context.Context) (bool, error) { + var overallEtcdPodCount int + if etcdEndpoints, overallEtcdPodCount, lastErr = getRawEtcdEndpointsFromPodAnnotationWithoutRetry(client); lastErr != nil { + return false, nil + } + if len(etcdEndpoints) == 0 || overallEtcdPodCount != len(etcdEndpoints) { + klog.V(4).Infof("found a total of %d etcd pods and the following endpoints: %v; retrying", + overallEtcdPodCount, etcdEndpoints) + return false, nil + } + return true, nil + }) + if err != nil { + const message = "could not retrieve the list of etcd endpoints" + if lastErr != nil { + return []string{}, errors.Wrap(lastErr, message) + } + return []string{}, errors.Wrap(err, message) + } + return etcdEndpoints, nil +} + +// getRawEtcdEndpointsFromPodAnnotationWithoutRetry returns the list of etcd endpoints as reported by etcd Pod annotations, +// along with the number of global etcd pods. This allows for callers to tell the difference between "no endpoints found", +// and "no endpoints found and pods were listed", so they can skip retrying. +func getRawEtcdEndpointsFromPodAnnotationWithoutRetry(client clientset.Interface) ([]string, int, error) { + klog.V(3).Infof("retrieving etcd endpoints from %q annotation in etcd Pods", constants.EtcdAdvertiseClientUrlsAnnotationKey) + podList, err := client.CoreV1().Pods(metav1.NamespaceSystem).List( + context.TODO(), + metav1.ListOptions{ + LabelSelector: fmt.Sprintf("component=%s,tier=%s", constants.Etcd, constants.ControlPlaneTier), + }, + ) + if err != nil { + return []string{}, 0, err + } + etcdEndpoints := []string{} + for _, pod := range podList.Items { + podIsReady := false + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + podIsReady = true + break + } + } + if !podIsReady { + klog.V(3).Infof("etcd pod %q is not ready", pod.ObjectMeta.Name) + } + etcdEndpoint, ok := pod.ObjectMeta.Annotations[constants.EtcdAdvertiseClientUrlsAnnotationKey] + if !ok { + klog.V(3).Infof("etcd Pod %q is missing the %q annotation; cannot infer etcd advertise client URL using the Pod annotation", pod.ObjectMeta.Name, constants.EtcdAdvertiseClientUrlsAnnotationKey) + continue + } + etcdEndpoints = append(etcdEndpoints, etcdEndpoint) + } + return etcdEndpoints, len(podList.Items), nil +} + +// Sync synchronizes client's endpoints with the known endpoints from the etcd membership. +func (c *Client) Sync() error { + // Syncs the list of endpoints + var cli etcdClient + var lastError error + err := wait.PollUntilContextTimeout(context.Background(), constants.EtcdAPICallRetryInterval, constants.EtcdAPICallTimeout, + true, func(_ context.Context) (bool, error) { + var err error + cli, err = c.newEtcdClient(c.Endpoints) + if err != nil { + lastError = err + return false, nil + } + defer func() { _ = cli.Close() }() + ctx, cancel := context.WithTimeout(context.Background(), etcdTimeout) + err = cli.Sync(ctx) + cancel() + if err == nil { + return true, nil + } + klog.V(5).Infof("Failed to sync etcd endpoints: %v", err) + lastError = err + return false, nil + }) + if err != nil { + return lastError + } + klog.V(1).Infof("etcd endpoints read from etcd: %s", strings.Join(cli.Endpoints(), ",")) + + c.Endpoints = cli.Endpoints() + return nil +} + +// Member struct defines an etcd member; it is used for avoiding to spread github.com/coreos/etcd dependency +// across kubeadm codebase +type Member struct { + Name string + PeerURL string +} + +func (c *Client) listMembers(timeout time.Duration) (*clientv3.MemberListResponse, error) { + // Gets the member list + var lastError error + var resp *clientv3.MemberListResponse + if timeout == 0 { + timeout = constants.EtcdAPICallTimeout + } + err := wait.PollUntilContextTimeout(context.Background(), constants.EtcdAPICallRetryInterval, timeout, + true, func(_ context.Context) (bool, error) { + cli, err := c.newEtcdClient(c.Endpoints) + if err != nil { + lastError = err + return false, nil + } + defer func() { _ = cli.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), etcdTimeout) + resp, err = cli.MemberList(ctx) + cancel() + if err == nil { + return true, nil + } + klog.V(5).Infof("Failed to get etcd member list: %v", err) + lastError = err + return false, nil + }) + if err != nil { + return nil, lastError + } + return resp, nil +} + +// GetMemberID returns the member ID of the given peer URL +func (c *Client) GetMemberID(peerURL string) (uint64, error) { + resp, err := c.listMembersFunc(0) + if err != nil { + return 0, err + } + + for _, member := range resp.Members { + if member.GetPeerURLs()[0] == peerURL { + return member.GetID(), nil + } + } + return 0, ErrNoMemberIDForPeerURL +} + +// ListMembers returns the member list. +func (c *Client) ListMembers() ([]Member, error) { + resp, err := c.listMembersFunc(0) + if err != nil { + return nil, err + } + + ret := make([]Member, 0, len(resp.Members)) + for _, m := range resp.Members { + ret = append(ret, Member{Name: m.Name, PeerURL: m.PeerURLs[0]}) + } + return ret, nil +} + +// RemoveMember notifies an etcd cluster to remove an existing member +func (c *Client) RemoveMember(id uint64) ([]Member, error) { + // Remove an existing member from the cluster + var lastError error + var resp *clientv3.MemberRemoveResponse + err := wait.PollUntilContextTimeout(context.Background(), constants.EtcdAPICallRetryInterval, constants.EtcdAPICallTimeout, + true, func(_ context.Context) (bool, error) { + cli, err := c.newEtcdClient(c.Endpoints) + if err != nil { + lastError = err + return false, nil + } + defer func() { _ = cli.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), etcdTimeout) + resp, err = cli.MemberRemove(ctx, id) + cancel() + if err == nil { + return true, nil + } + if errors.Is(rpctypes.ErrMemberNotFound, err) { + klog.V(5).Infof("Member was already removed, because member %s was not found", strconv.FormatUint(id, 16)) + return true, nil + } + klog.V(5).Infof("Failed to remove etcd member: %v", err) + lastError = err + return false, nil + }) + if err != nil { + return nil, lastError + } + + // Returns the updated list of etcd members + ret := []Member{} + if resp != nil { + for _, m := range resp.Members { + ret = append(ret, Member{Name: m.Name, PeerURL: m.PeerURLs[0]}) + } + + } + + return ret, nil +} + +// AddMember adds a new member into the etcd cluster +func (c *Client) AddMember(name string, peerAddrs string) ([]Member, error) { + return c.addMember(name, peerAddrs, false) +} + +// AddMemberAsLearner adds a new learner member into the etcd cluster. +func (c *Client) AddMemberAsLearner(name string, peerAddrs string) ([]Member, error) { + return c.addMember(name, peerAddrs, true) +} + +// addMember notifies an existing etcd cluster that a new member is joining, and +// return the updated list of members. If the member has already been added to the +// cluster, this will return the existing list of etcd members. +func (c *Client) addMember(name string, peerAddrs string, isLearner bool) ([]Member, error) { + // Parse the peer address, required to add the client URL later to the list + // of endpoints for this client. Parsing as a first operation to make sure that + // if this fails no member addition is performed on the etcd cluster. + parsedPeerAddrs, err := url.Parse(peerAddrs) + if err != nil { + return nil, errors.Wrapf(err, "error parsing peer address %s", peerAddrs) + } + + cli, err := c.newEtcdClient(c.Endpoints) + if err != nil { + return nil, err + } + defer func() { _ = cli.Close() }() + + // Adds a new member to the cluster + var ( + lastError error + respMembers []*etcdserverpb.Member + learnerID uint64 + resp *clientv3.MemberAddResponse + ) + err = wait.PollUntilContextTimeout(context.Background(), constants.EtcdAPICallRetryInterval, constants.EtcdAPICallTimeout, + true, func(_ context.Context) (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), etcdTimeout) + defer cancel() + if isLearner { + // if learnerID is set, it means the etcd member is already added successfully. + if learnerID == 0 { + klog.V(1).Info("[etcd] Adding etcd member as learner") + resp, err = cli.MemberAddAsLearner(ctx, []string{peerAddrs}) + if err != nil { + lastError = err + return false, nil + } + learnerID = resp.Member.ID + } + respMembers = resp.Members + return true, nil + } + + resp, err = cli.MemberAdd(ctx, []string{peerAddrs}) + if err == nil { + respMembers = resp.Members + return true, nil + } + + // If the error indicates that the peer already exists, exit early. In this situation, resp is nil, so + // call out to MemberList to fetch all the members before returning. + if errors.Is(err, rpctypes.ErrPeerURLExist) { + klog.V(5).Info("The peer URL for the added etcd member already exists. Fetching the existing etcd members") + var listResp *clientv3.MemberListResponse + listResp, err = cli.MemberList(ctx) + if err == nil { + respMembers = listResp.Members + return true, nil + } + } + + klog.V(5).Infof("Failed to add etcd member: %v", err) + lastError = err + return false, nil + }) + if err != nil { + return nil, lastError + } + + // Returns the updated list of etcd members + ret := []Member{} + for _, m := range respMembers { + // If the peer address matches, this is the member we are adding. + // Use the name we passed to the function. + if peerAddrs == m.PeerURLs[0] { + ret = append(ret, Member{Name: name, PeerURL: peerAddrs}) + continue + } + // Otherwise, we are processing other existing etcd members returned by AddMembers. + memberName := m.Name + // In some cases during concurrent join, some members can end up without a name. + // Use the member ID as name for those. + if len(memberName) == 0 { + memberName = strconv.FormatUint(m.ID, 16) + } + ret = append(ret, Member{Name: memberName, PeerURL: m.PeerURLs[0]}) + } + + // Add the new member client address to the list of endpoints + c.Endpoints = append(c.Endpoints, GetClientURLByIP(parsedPeerAddrs.Hostname())) + + return ret, nil +} + +// isLearner returns true if the given member ID is a learner. +func (c *Client) isLearner(memberID uint64) (bool, error) { + resp, err := c.listMembersFunc(0) + if err != nil { + return false, err + } + + for _, member := range resp.Members { + if member.ID == memberID && member.IsLearner { + return true, nil + } + } + return false, nil +} + +// MemberPromote promotes a member as a voting member. If the given member ID is already a voting member this method +// will return early and do nothing. +func (c *Client) MemberPromote(learnerID uint64) error { + isLearner, err := c.isLearner(learnerID) + if err != nil { + return err + } + if !isLearner { + klog.V(1).Infof("[etcd] Member %s already promoted.", strconv.FormatUint(learnerID, 16)) + return nil + } + + klog.V(1).Infof("[etcd] Promoting a learner as a voting member: %s", strconv.FormatUint(learnerID, 16)) + cli, err := c.newEtcdClient(c.Endpoints) + if err != nil { + return err + } + defer func() { _ = cli.Close() }() + + // TODO: warning logs from etcd client should be removed. + // The warning logs are printed by etcd client code for several reasons, including + // 1. can not promote yet(no synced) + // 2. context deadline exceeded + // 3. peer URLs already exists + // Once the client provides a way to check if the etcd learner is ready to promote, the retry logic can be revisited. + var ( + lastError error + ) + err = wait.PollUntilContextTimeout(context.Background(), constants.EtcdAPICallRetryInterval, constants.EtcdAPICallTimeout, + true, func(_ context.Context) (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), etcdTimeout) + defer cancel() + + _, err = cli.MemberPromote(ctx, learnerID) + if err == nil { + klog.V(1).Infof("[etcd] The learner was promoted as a voting member: %s", strconv.FormatUint(learnerID, 16)) + return true, nil + } + klog.V(5).Infof("[etcd] Promoting the learner %s failed: %v", strconv.FormatUint(learnerID, 16), err) + lastError = err + return false, nil + }) + if err != nil { + return lastError + } + return nil +} + +// CheckClusterHealth returns nil for status Up or error for status Down +func (c *Client) CheckClusterHealth() error { + _, err := c.getClusterStatus() + return err +} + +// getClusterStatus returns nil for status Up (along with endpoint status response map) or error for status Down +func (c *Client) getClusterStatus() (map[string]*clientv3.StatusResponse, error) { + clusterStatus := make(map[string]*clientv3.StatusResponse) + for _, ep := range c.Endpoints { + // Gets the member status + var lastError error + var resp *clientv3.StatusResponse + err := wait.PollUntilContextTimeout(context.Background(), constants.EtcdAPICallRetryInterval, constants.EtcdAPICallTimeout, + true, func(_ context.Context) (bool, error) { + cli, err := c.newEtcdClient(c.Endpoints) + if err != nil { + lastError = err + return false, nil + } + defer func() { _ = cli.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), etcdTimeout) + resp, err = cli.Status(ctx, ep) + cancel() + if err == nil { + return true, nil + } + klog.V(5).Infof("Failed to get etcd status for %s: %v", ep, err) + lastError = err + return false, nil + }) + if err != nil { + return nil, lastError + } + + clusterStatus[ep] = resp + } + return clusterStatus, nil +} + +// WaitForClusterAvailable returns true if all endpoints in the cluster are available after retry attempts, an error is returned otherwise +func (c *Client) WaitForClusterAvailable(retries int, retryInterval time.Duration) (bool, error) { + for i := 0; i < retries; i++ { + if i > 0 { + klog.V(1).Infof("[etcd] Waiting %v until next retry\n", retryInterval) + time.Sleep(retryInterval) + } + klog.V(2).Infof("[etcd] attempting to see if all cluster endpoints (%s) are available %d/%d", c.Endpoints, i+1, retries) + _, err := c.getClusterStatus() + if err != nil { + switch err { + case context.DeadlineExceeded: + klog.V(1).Infof("[etcd] Attempt timed out") + default: + klog.V(1).Infof("[etcd] Attempt failed with error: %v\n", err) + } + continue + } + return true, nil + } + return false, errors.New("timeout waiting for etcd cluster to be available") +} + +// GetClientURL creates an HTTPS URL that uses the configured advertise +// address and client port for the API controller +func GetClientURL(localEndpoint *kubeadmapi.APIEndpoint) string { + return "https://" + net.JoinHostPort(localEndpoint.AdvertiseAddress, strconv.Itoa(constants.EtcdListenClientPort)) +} + +// GetPeerURL creates an HTTPS URL that uses the configured advertise +// address and peer port for the API controller +func GetPeerURL(localEndpoint *kubeadmapi.APIEndpoint) string { + return "https://" + net.JoinHostPort(localEndpoint.AdvertiseAddress, strconv.Itoa(constants.EtcdListenPeerPort)) +} + +// GetClientURLByIP creates an HTTPS URL based on an IP address +// and the client listening port. +func GetClientURLByIP(ip string) string { + return "https://" + net.JoinHostPort(ip, strconv.Itoa(constants.EtcdListenClientPort)) +} diff --git a/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/testdata/manifests/etcd b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/testdata/manifests/etcd new file mode 100644 index 0000000000..6cb772bfcf --- /dev/null +++ b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/testdata/manifests/etcd @@ -0,0 +1,83 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + kubeadm.kubernetes.io/etcd.advertise-client-urls: https://195.17.37.116:2379 + creationTimestamp: null + labels: + component: etcd + tier: control-plane + name: etcd + namespace: kube-system +spec: + containers: + - command: + - etcd + - --advertise-client-urls=https://195.17.37.116:2379 + - --cert-file=/var/lib/kubeadm/pki/etcd/server.crt + - --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - --client-cert-auth=true + - --data-dir=/var/lib/etcd + - --experimental-initial-corrupt-check=true + - --experimental-watch-progress-notify-interval=5s + - --initial-advertise-peer-urls=https://195.17.37.116:2380 + - --initial-cluster=eksa-test-67419a2-w6kk5=https://195.17.37.116:2380 + - --key-file=/var/lib/kubeadm/pki/etcd/server.key + - --listen-client-urls=https://127.0.0.1:2379,https://195.17.37.116:2379 + - --listen-metrics-urls=http://127.0.0.1:2381 + - --listen-peer-urls=https://195.17.37.116:2380 + - --name=eksa-test-67419a2-w6kk5 + - --peer-cert-file=/var/lib/kubeadm/pki/etcd/peer.crt + - --peer-client-cert-auth=true + - --peer-key-file=/var/lib/kubeadm/pki/etcd/peer.key + - --peer-trusted-ca-file=/var/lib/kubeadm/pki/etcd/ca.crt + - --snapshot-count=10000 + - --trusted-ca-file=/var/lib/kubeadm/pki/etcd/ca.crt + image: public.ecr.aws/eks-distro/etcd-io/etcd:v3.5.10-eks-1-28-15 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 8 + httpGet: + host: 127.0.0.1 + path: /health?exclude=NOSPACE&serializable=true + port: 2381 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + name: etcd + resources: + requests: + cpu: 100m + memory: 100Mi + startupProbe: + failureThreshold: 24 + httpGet: + host: 127.0.0.1 + path: /health?serializable=false + port: 2381 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + volumeMounts: + - mountPath: /var/lib/etcd + name: etcd-data + - mountPath: /var/lib/kubeadm/pki/etcd + name: etcd-certs + hostNetwork: true + priority: 2000001000 + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault + volumes: + - hostPath: + path: /var/lib/kubeadm/pki/etcd + type: DirectoryOrCreate + name: etcd-certs + - hostPath: + path: /var/lib/etcd + type: DirectoryOrCreate + name: etcd-data +status: {} diff --git a/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/testdata/manifests/kube-apiserver b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/testdata/manifests/kube-apiserver new file mode 100644 index 0000000000..6e3198154f --- /dev/null +++ b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/testdata/manifests/kube-apiserver @@ -0,0 +1,133 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: 195.17.37.116:6443 + creationTimestamp: null + labels: + component: kube-apiserver + tier: control-plane + name: kube-apiserver + namespace: kube-system +spec: + containers: + - command: + - kube-apiserver + - --advertise-address=195.17.37.116 + - --allow-privileged=true + - --audit-log-maxage=30 + - --audit-log-maxbackup=10 + - --audit-log-maxsize=512 + - --audit-log-path=/var/log/kubernetes/api-audit.log + - --audit-policy-file=/etc/kubernetes/audit-policy.yaml + - --authorization-mode=Node,RBAC + - --client-ca-file=/var/lib/kubeadm/pki/ca.crt + - --cloud-provider=external + - --enable-admission-plugins=NodeRestriction + - --enable-bootstrap-token-auth=true + - --etcd-cafile=/var/lib/kubeadm/pki/etcd/ca.crt + - --etcd-certfile=/var/lib/kubeadm/pki/apiserver-etcd-client.crt + - --etcd-keyfile=/var/lib/kubeadm/pki/apiserver-etcd-client.key + - --etcd-servers=https://127.0.0.1:2379 + - --kubelet-client-certificate=/var/lib/kubeadm/pki/apiserver-kubelet-client.crt + - --kubelet-client-key=/var/lib/kubeadm/pki/apiserver-kubelet-client.key + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --profiling=false + - --proxy-client-cert-file=/var/lib/kubeadm/pki/front-proxy-client.crt + - --proxy-client-key-file=/var/lib/kubeadm/pki/front-proxy-client.key + - --requestheader-allowed-names=front-proxy-client + - --requestheader-client-ca-file=/var/lib/kubeadm/pki/front-proxy-ca.crt + - --requestheader-extra-headers-prefix=X-Remote-Extra- + - --requestheader-group-headers=X-Remote-Group + - --requestheader-username-headers=X-Remote-User + - --secure-port=6443 + - --service-account-issuer=https://kubernetes.default.svc.cluster.local + - --service-account-key-file=/var/lib/kubeadm/pki/sa.pub + - --service-account-signing-key-file=/var/lib/kubeadm/pki/sa.key + - --service-cluster-ip-range=10.96.0.0/12 + - --tls-cert-file=/var/lib/kubeadm/pki/apiserver.crt + - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - --tls-private-key-file=/var/lib/kubeadm/pki/apiserver.key + env: + - name: NO_PROXY + value: localhost,127.0.0.1,.cluster.local + - name: no_proxy + value: localhost,127.0.0.1,.cluster.local + image: public.ecr.aws/eks-distro/kubernetes/kube-apiserver:v1.28.5-eks-1-28-15 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 8 + httpGet: + host: 195.17.37.116 + path: /livez + port: 6443 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + name: kube-apiserver + readinessProbe: + failureThreshold: 3 + httpGet: + host: 195.17.37.116 + path: /readyz + port: 6443 + scheme: HTTPS + periodSeconds: 1 + timeoutSeconds: 15 + resources: + requests: + cpu: 250m + startupProbe: + failureThreshold: 24 + httpGet: + host: 195.17.37.116 + path: /livez + port: 6443 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + volumeMounts: + - mountPath: /var/log/kubernetes + name: audit-log-dir + - mountPath: /etc/kubernetes/audit-policy.yaml + name: audit-policy + readOnly: true + - mountPath: /etc/ssl/certs + name: ca-certs + readOnly: true + - mountPath: /etc/pki + name: etc-pki + readOnly: true + - mountPath: /var/lib/kubeadm/pki + name: k8s-certs + readOnly: true + hostNetwork: true + priority: 2000001000 + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault + volumes: + - hostPath: + path: /var/log/kubernetes + type: DirectoryOrCreate + name: audit-log-dir + - hostPath: + path: /var/lib/kubeadm/audit-policy.yaml + type: File + name: audit-policy + - hostPath: + path: /etc/ssl/certs + type: DirectoryOrCreate + name: ca-certs + - hostPath: + path: /etc/pki + type: DirectoryOrCreate + name: etc-pki + - hostPath: + path: /var/lib/kubeadm/pki + type: DirectoryOrCreate + name: k8s-certs +status: {} diff --git a/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/testdata/manifests/kube-vip b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/testdata/manifests/kube-vip new file mode 100644 index 0000000000..eba93e346e --- /dev/null +++ b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/testdata/manifests/kube-vip @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system +spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: address + value: 105.16.99.106 + image: public.ecr.aws/l0g8r8j6/kube-vip/kube-vip:v0.7.0-eks-a-v0.19.0-dev-build.30 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostNetwork: true + volumes: + - hostPath: + path: /var/lib/kubeadm/admin.conf + type: File + name: kubeconfig +status: {} diff --git a/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/utils.go b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/utils.go index 55387fd18d..6826157894 100644 --- a/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/utils.go +++ b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/utils.go @@ -14,8 +14,11 @@ import ( "github.com/eks-anywhere-build-tooling/aws/bottlerocket-bootstrap/pkg/files" "github.com/eks-anywhere-build-tooling/aws/bottlerocket-bootstrap/pkg/utils" + "github.com/eks-anywhere-build-tooling/aws/bottlerocket-bootstrap/pkg/kubeadm/etcd" "github.com/pkg/errors" "sigs.k8s.io/yaml" + clientv3 "go.etcd.io/etcd/client/v3" + // etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" ) const ( @@ -337,3 +340,38 @@ func patchKubeVipManifest() error { return nil } + +type EtcdClient interface { + + // MemberList lists the current cluster membership. + MemberList(ctx context.Context) (*clientv3.MemberListResponse, error) + + // MemberPromote promotes a member from raft learner (non-voting) to raft voting member. + MemberPromote(ctx context.Context, id uint64) (*clientv3.MemberPromoteResponse, error) +} + +func NewEtcdClient(certificatesDir string, manifestsDir string) (*etcd.Client, error) { + // ToDo: use the certificates directory to read certs/keys into variables and pass to client + endpoints := []string{} // extract endpoints from the etcd static pod manifests, rip off the code in pods.go EnableStaticPods if you need to + client, err := etcd.New(endpoints, "", "", "") + if err != nil { + return nil, err + } + return client, nil +} + +// func promoteEtcdLearner(client clientset.Interface, endpoint *kubeadmapi.APIEndpoint, certificatesDir string) error { +func promoteEtcdLearner(client etcd.Client, peerUrl string) error { + + + memberID, err := client.GetMemberID(peerUrl) + if err != nil { + return err + } + + err = client.MemberPromote(memberID) + if err != nil { + return err + } + return nil +} \ No newline at end of file diff --git a/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/utils_test.go b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/utils_test.go index 585852639d..b049fc275b 100644 --- a/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/utils_test.go +++ b/projects/aws/bottlerocket-bootstrap/pkg/kubeadm/utils_test.go @@ -3,8 +3,10 @@ package kubeadm import ( "os" "testing" + "fmt" "github.com/google/go-cmp/cmp" + "github.com/eks-anywhere-build-tooling/aws/bottlerocket-bootstrap/pkg/utils" ) const localEtcdClusterConf = `'apiServer: @@ -215,3 +217,18 @@ func TestReadKubeletTlsConfig(t *testing.T) { }) } } + +func TestCreateEtcdClient(t *testing.T) { + + podDefinitions, err := utils.EnableStaticPods("testdata/manifests/") + if err != nil { + t.Fail() + // t.Fail(errors.Wrap(err, "Error enabling static pods")) + } + for _, p := range podDefinitions { + fmt.Println(p.Kind) + fmt.Println(p.Annotations["kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint"]) + } + + t.Fail() +} \ No newline at end of file diff --git a/projects/aws/bottlerocket-bootstrap/pkg/utils/pods.go b/projects/aws/bottlerocket-bootstrap/pkg/utils/pods.go index 6e54a16672..8d9183f07f 100644 --- a/projects/aws/bottlerocket-bootstrap/pkg/utils/pods.go +++ b/projects/aws/bottlerocket-bootstrap/pkg/utils/pods.go @@ -56,7 +56,7 @@ func EnableStaticPods(path string) ([]*v1.Pod, error) { return podDefinitions, nil } -var podFileExtensions = map[string]struct{}{".yaml": {}, ".manifest": {}} +var podFileExtensions = map[string]struct{}{".yaml": {}, ".manifest": {}, "": {}} func isPodFile(f fs.FileInfo) bool { _, ok := podFileExtensions[filepath.Ext(f.Name())]