From 1484b5ff14e46ddca9e3fe2043d6bdc97ba4dc1a Mon Sep 17 00:00:00 2001 From: Nick Baker Date: Fri, 4 Oct 2024 11:34:55 -0700 Subject: [PATCH] nodeadm(feat): add mime multi-part support to file config provider (#1992) --- .../node.eks.aws_v1alpha1_nodeconfig.yaml | 23 ---- nodeadm/internal/api/merge_test.go | 6 + nodeadm/internal/configprovider/encoding.go | 42 ++++++ nodeadm/internal/configprovider/file.go | 7 +- nodeadm/internal/configprovider/mime.go | 96 ++++++++++++++ nodeadm/internal/configprovider/userdata.go | 122 +----------------- .../internal/configprovider/userdata_test.go | 9 +- .../aemm-inf1-config.json | 0 .../containerd-runtime-config-neuron/run.sh | 2 +- .../aemm-g5-config.json | 0 .../containerd-runtime-config-nvidia/run.sh | 2 +- .../e2e/cases/mime-userdata-merge/config.yaml | 41 ++++++ .../expected-kubelet-config.json | 70 ++++++++++ .../test/e2e/cases/mime-userdata-merge/run.sh | 17 +++ nodeadm/test/e2e/infra/Dockerfile | 2 - 15 files changed, 282 insertions(+), 157 deletions(-) delete mode 100644 nodeadm/example/node.eks.aws_v1alpha1_nodeconfig.yaml create mode 100644 nodeadm/internal/configprovider/encoding.go create mode 100644 nodeadm/internal/configprovider/mime.go rename nodeadm/test/e2e/{infra => cases/containerd-runtime-config-neuron}/aemm-inf1-config.json (100%) rename nodeadm/test/e2e/{infra => cases/containerd-runtime-config-nvidia}/aemm-g5-config.json (100%) create mode 100644 nodeadm/test/e2e/cases/mime-userdata-merge/config.yaml create mode 100644 nodeadm/test/e2e/cases/mime-userdata-merge/expected-kubelet-config.json create mode 100755 nodeadm/test/e2e/cases/mime-userdata-merge/run.sh diff --git a/nodeadm/example/node.eks.aws_v1alpha1_nodeconfig.yaml b/nodeadm/example/node.eks.aws_v1alpha1_nodeconfig.yaml deleted file mode 100644 index 300fcf2ec..000000000 --- a/nodeadm/example/node.eks.aws_v1alpha1_nodeconfig.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -apiVersion: node.eks.aws/v1alpha1 -kind: NodeConfig -metadata: - name: example -spec: - cluster: - name: my-cluster - apiServerEndpoint: https://example.com - certificateAuthority: Y2VydGlmaWNhdGVBdXRob3JpdHk= - cidr: 10.100.0.0/16 - kubelet: - config: - kind: KubeletConfiguration - apiVersion: kubelet.config.k8s.io/v1beta1 - maxPods: 42 - clusterDNS: - - 0.0.0.0 - - 1.1.1.1 - flags: - - --v=2 - - --register-with-taints=foo=bar:NoSchedule - - --node-labels=foo=bar diff --git a/nodeadm/internal/api/merge_test.go b/nodeadm/internal/api/merge_test.go index a59105574..ee2b82076 100644 --- a/nodeadm/internal/api/merge_test.go +++ b/nodeadm/internal/api/merge_test.go @@ -95,6 +95,9 @@ func TestMerge(t *testing.T) { "verbosity": 5, }, "podsPerCore": 20, + "systemReserved": map[string]interface{}{ + "cpu": "150m", + }, }), Flags: []string{ "--node-labels=nodegroup=example", @@ -167,6 +170,9 @@ discard_unpacked_layers = false`), }, "maxPods": 150, "podsPerCore": 20, + "systemReserved": map[string]interface{}{ + "cpu": "150m", + }, }), Flags: []string{ "--node-labels=nodegroup=example", diff --git a/nodeadm/internal/configprovider/encoding.go b/nodeadm/internal/configprovider/encoding.go new file mode 100644 index 000000000..73ad4e514 --- /dev/null +++ b/nodeadm/internal/configprovider/encoding.go @@ -0,0 +1,42 @@ +package configprovider + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "fmt" + "io" +) + +func decodeIfBase64(data []byte) ([]byte, error) { + e := base64.StdEncoding + maxDecodedLen := e.DecodedLen(len(data)) + decodedData := make([]byte, maxDecodedLen) + decodedLen, err := e.Decode(decodedData, data) + if err != nil { + return data, nil + } + return decodedData[:decodedLen], nil +} + +// https://en.wikipedia.org/wiki/Gzip +const gzipMagicNumber = uint16(0x1f8b) + +func decompressIfGZIP(data []byte) ([]byte, error) { + if len(data) < 2 { + return data, nil + } + preamble := uint16(data[0])<<8 | uint16(data[1]) + if preamble == gzipMagicNumber { + reader, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("failed to create GZIP reader: %v", err) + } + if decompressed, err := io.ReadAll(reader); err != nil { + return nil, fmt.Errorf("failed to read from GZIP reader: %v", err) + } else { + return decompressed, nil + } + } + return data, nil +} diff --git a/nodeadm/internal/configprovider/file.go b/nodeadm/internal/configprovider/file.go index b4ed09892..3d8fb4411 100644 --- a/nodeadm/internal/configprovider/file.go +++ b/nodeadm/internal/configprovider/file.go @@ -6,7 +6,6 @@ import ( "os" internalapi "github.com/awslabs/amazon-eks-ami/nodeadm/internal/api" - apibridge "github.com/awslabs/amazon-eks-ami/nodeadm/internal/api/bridge" ) type fileConfigProvider struct { @@ -36,9 +35,5 @@ func (fcs *fileConfigProvider) Provide() (*internalapi.NodeConfig, error) { if err != nil { return nil, err } - config, err := apibridge.DecodeNodeConfig(data) - if err != nil { - return nil, err - } - return config, nil + return parseMaybeMultipart(data) } diff --git a/nodeadm/internal/configprovider/mime.go b/nodeadm/internal/configprovider/mime.go new file mode 100644 index 000000000..8fbc4faf1 --- /dev/null +++ b/nodeadm/internal/configprovider/mime.go @@ -0,0 +1,96 @@ +package configprovider + +import ( + "bytes" + "fmt" + "io" + "mime" + "mime/multipart" + "net/mail" + "strings" + + internalapi "github.com/awslabs/amazon-eks-ami/nodeadm/internal/api" + apibridge "github.com/awslabs/amazon-eks-ami/nodeadm/internal/api/bridge" +) + +func parseMaybeMultipart(data []byte) (*internalapi.NodeConfig, error) { + // if the MIME data fails to parse as a multipart document, then fall back + // to parsing the entire userdata as the node config. + if multipartReader, err := getMultipartReader(data); err == nil { + config, err := parseMultipart(multipartReader) + if err != nil { + return nil, err + } + return config, nil + } else { + config, err := apibridge.DecodeNodeConfig(data) + if err != nil { + return nil, err + } + return config, nil + } +} + +func parseMultipart(userDataReader *multipart.Reader) (*internalapi.NodeConfig, error) { + var nodeConfigs []*internalapi.NodeConfig + for { + part, err := userDataReader.NextPart() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if partHeader := part.Header.Get(contentTypeHeader); len(partHeader) > 0 { + mediaType, _, err := mime.ParseMediaType(partHeader) + if err != nil { + return nil, err + } + if mediaType == nodeConfigMediaType { + nodeConfigPart, err := io.ReadAll(part) + if err != nil { + return nil, err + } + nodeConfigPart, err = decodeIfBase64(nodeConfigPart) + if err != nil { + return nil, err + } + nodeConfigPart, err = decompressIfGZIP(nodeConfigPart) + if err != nil { + return nil, err + } + decodedConfig, err := apibridge.DecodeNodeConfig(nodeConfigPart) + if err != nil { + return nil, err + } + nodeConfigs = append(nodeConfigs, decodedConfig) + } + } + } + if len(nodeConfigs) > 0 { + var config = nodeConfigs[0] + for _, nodeConfig := range nodeConfigs[1:] { + if err := config.Merge(nodeConfig); err != nil { + return nil, err + } + } + return config, nil + } else { + return nil, fmt.Errorf("could not find NodeConfig within UserData") + } +} + +func getMultipartReader(data []byte) (*multipart.Reader, error) { + msg, err := mail.ReadMessage(bytes.NewReader(data)) + if err != nil { + return nil, err + } + mediaType, params, err := mime.ParseMediaType(msg.Header.Get(contentTypeHeader)) + if err != nil { + return nil, err + } + if !strings.HasPrefix(mediaType, multipartContentTypePrefix) { + return nil, fmt.Errorf("MIME type is not multipart") + } + return multipart.NewReader(msg.Body, params[mimeBoundaryParam]), nil +} diff --git a/nodeadm/internal/configprovider/userdata.go b/nodeadm/internal/configprovider/userdata.go index 671f8881f..55f1ebce6 100644 --- a/nodeadm/internal/configprovider/userdata.go +++ b/nodeadm/internal/configprovider/userdata.go @@ -1,19 +1,10 @@ package configprovider import ( - "bytes" - "compress/gzip" - "encoding/base64" "fmt" - "io" - "mime" - "mime/multipart" - "net/mail" - "strings" "github.com/awslabs/amazon-eks-ami/nodeadm/api" internalapi "github.com/awslabs/amazon-eks-ami/nodeadm/internal/api" - apibridge "github.com/awslabs/amazon-eks-ami/nodeadm/internal/api/bridge" imds "github.com/awslabs/amazon-eks-ami/nodeadm/internal/aws/imds" ) @@ -57,116 +48,5 @@ func (p *userDataConfigProvider) Provide() (*internalapi.NodeConfig, error) { if err != nil { return nil, fmt.Errorf("failed to decompress user data: %v", err) } - // if the MIME data fails to parse as a multipart document, then fall back - // to parsing the entire userdata as the node config. - if multipartReader, err := getMIMEMultipartReader(userData); err == nil { - config, err := parseMultipart(multipartReader) - if err != nil { - return nil, err - } - return config, nil - } else { - config, err := apibridge.DecodeNodeConfig(userData) - if err != nil { - return nil, err - } - return config, nil - } -} - -func getMIMEMultipartReader(data []byte) (*multipart.Reader, error) { - msg, err := mail.ReadMessage(bytes.NewReader(data)) - if err != nil { - return nil, err - } - mediaType, params, err := mime.ParseMediaType(msg.Header.Get(contentTypeHeader)) - if err != nil { - return nil, err - } - if !strings.HasPrefix(mediaType, multipartContentTypePrefix) { - return nil, fmt.Errorf("MIME type is not multipart") - } - return multipart.NewReader(msg.Body, params[mimeBoundaryParam]), nil -} - -func parseMultipart(userDataReader *multipart.Reader) (*internalapi.NodeConfig, error) { - var nodeConfigs []*internalapi.NodeConfig - for { - part, err := userDataReader.NextPart() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - if partHeader := part.Header.Get(contentTypeHeader); len(partHeader) > 0 { - mediaType, _, err := mime.ParseMediaType(partHeader) - if err != nil { - return nil, err - } - if mediaType == nodeConfigMediaType { - nodeConfigPart, err := io.ReadAll(part) - if err != nil { - return nil, err - } - nodeConfigPart, err = decodeIfBase64(nodeConfigPart) - if err != nil { - return nil, err - } - nodeConfigPart, err = decompressIfGZIP(nodeConfigPart) - if err != nil { - return nil, err - } - decodedConfig, err := apibridge.DecodeNodeConfig(nodeConfigPart) - if err != nil { - return nil, err - } - nodeConfigs = append(nodeConfigs, decodedConfig) - } - } - } - if len(nodeConfigs) > 0 { - var config = nodeConfigs[0] - for _, nodeConfig := range nodeConfigs[1:] { - if err := config.Merge(nodeConfig); err != nil { - return nil, err - } - } - return config, nil - } else { - return nil, fmt.Errorf("could not find NodeConfig within UserData") - } -} - -func decodeIfBase64(data []byte) ([]byte, error) { - e := base64.StdEncoding - maxDecodedLen := e.DecodedLen(len(data)) - decodedData := make([]byte, maxDecodedLen) - decodedLen, err := e.Decode(decodedData, data) - if err != nil { - return data, nil - } - return decodedData[:decodedLen], nil -} - -// https://en.wikipedia.org/wiki/Gzip -const gzipMagicNumber = uint16(0x1f8b) - -func decompressIfGZIP(data []byte) ([]byte, error) { - if len(data) < 2 { - return data, nil - } - preamble := uint16(data[0])<<8 | uint16(data[1]) - if preamble == gzipMagicNumber { - reader, err := gzip.NewReader(bytes.NewReader(data)) - if err != nil { - return nil, fmt.Errorf("failed to create GZIP reader: %v", err) - } - if decompressed, err := io.ReadAll(reader); err != nil { - return nil, fmt.Errorf("failed to read from GZIP reader: %v", err) - } else { - return decompressed, nil - } - } - return data, nil + return parseMaybeMultipart(userData) } diff --git a/nodeadm/internal/configprovider/userdata_test.go b/nodeadm/internal/configprovider/userdata_test.go index 60eadc6dc..3e948554b 100644 --- a/nodeadm/internal/configprovider/userdata_test.go +++ b/nodeadm/internal/configprovider/userdata_test.go @@ -101,6 +101,8 @@ func Test_Provide(t *testing.T) { " config:", " maxPods: 150", " podsPerCore: 20", + " systemReserved:", + " cpu: 150m", " flags:", " - --v=5", " - --node-labels=foo=baz", @@ -117,9 +119,10 @@ func Test_Provide(t *testing.T) { }, Kubelet: api.KubeletOptions{ Config: api.InlineDocument{ - "maxPods": runtime.RawExtension{Raw: []byte("150")}, - "podsPerCore": runtime.RawExtension{Raw: []byte("20")}, - "port": runtime.RawExtension{Raw: []byte("1010")}, + "maxPods": runtime.RawExtension{Raw: []byte("150")}, + "podsPerCore": runtime.RawExtension{Raw: []byte("20")}, + "port": runtime.RawExtension{Raw: []byte("1010")}, + "systemReserved": runtime.RawExtension{Raw: []byte(`{"cpu":"150m"}`)}, }, Flags: []string{ "--v=2", diff --git a/nodeadm/test/e2e/infra/aemm-inf1-config.json b/nodeadm/test/e2e/cases/containerd-runtime-config-neuron/aemm-inf1-config.json similarity index 100% rename from nodeadm/test/e2e/infra/aemm-inf1-config.json rename to nodeadm/test/e2e/cases/containerd-runtime-config-neuron/aemm-inf1-config.json diff --git a/nodeadm/test/e2e/cases/containerd-runtime-config-neuron/run.sh b/nodeadm/test/e2e/cases/containerd-runtime-config-neuron/run.sh index 61d129308..b19de3015 100755 --- a/nodeadm/test/e2e/cases/containerd-runtime-config-neuron/run.sh +++ b/nodeadm/test/e2e/cases/containerd-runtime-config-neuron/run.sh @@ -6,7 +6,7 @@ set -o pipefail source /helpers.sh -mock::aws /etc/aemm-inf1-config.json +mock::aws aemm-inf1-config.json mock::kubelet 1.27.0 wait::dbus-ready diff --git a/nodeadm/test/e2e/infra/aemm-g5-config.json b/nodeadm/test/e2e/cases/containerd-runtime-config-nvidia/aemm-g5-config.json similarity index 100% rename from nodeadm/test/e2e/infra/aemm-g5-config.json rename to nodeadm/test/e2e/cases/containerd-runtime-config-nvidia/aemm-g5-config.json diff --git a/nodeadm/test/e2e/cases/containerd-runtime-config-nvidia/run.sh b/nodeadm/test/e2e/cases/containerd-runtime-config-nvidia/run.sh index 758b8305c..98ed4c2de 100755 --- a/nodeadm/test/e2e/cases/containerd-runtime-config-nvidia/run.sh +++ b/nodeadm/test/e2e/cases/containerd-runtime-config-nvidia/run.sh @@ -6,7 +6,7 @@ set -o pipefail source /helpers.sh -mock::aws /etc/aemm-g5-config.json +mock::aws aemm-g5-config.json mock::kubelet 1.27.0 wait::dbus-ready diff --git a/nodeadm/test/e2e/cases/mime-userdata-merge/config.yaml b/nodeadm/test/e2e/cases/mime-userdata-merge/config.yaml new file mode 100644 index 000000000..a8af6cfcb --- /dev/null +++ b/nodeadm/test/e2e/cases/mime-userdata-merge/config.yaml @@ -0,0 +1,41 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: my-cluster + apiServerEndpoint: https://example.com + certificateAuthority: Y2VydGlmaWNhdGVBdXRob3JpdHk= + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 58 + clusterDNS: + - 10.100.0.10 + +--// +Content-Type: application/node.eks.aws + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + systemReserved: + cpu: 150m + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash + +echo "hello!" + +--//-- diff --git a/nodeadm/test/e2e/cases/mime-userdata-merge/expected-kubelet-config.json b/nodeadm/test/e2e/cases/mime-userdata-merge/expected-kubelet-config.json new file mode 100644 index 000000000..f3b614378 --- /dev/null +++ b/nodeadm/test/e2e/cases/mime-userdata-merge/expected-kubelet-config.json @@ -0,0 +1,70 @@ +{ + "kind": "KubeletConfiguration", + "apiVersion": "kubelet.config.k8s.io/v1beta1", + "address": "0.0.0.0", + "authentication": { + "x509": { + "clientCAFile": "/etc/kubernetes/pki/ca.crt" + }, + "webhook": { + "enabled": true, + "cacheTTL": "2m0s" + }, + "anonymous": { + "enabled": false + } + }, + "authorization": { + "mode": "Webhook", + "webhook": { + "cacheAuthorizedTTL": "5m0s", + "cacheUnauthorizedTTL": "30s" + } + }, + "cgroupDriver": "systemd", + "cgroupRoot": "/", + "clusterDomain": "cluster.local", + "containerRuntimeEndpoint": "unix:///run/containerd/containerd.sock", + "featureGates": { + "KubeletCredentialProviders": true, + "RotateKubeletServerCertificate": true + }, + "hairpinMode": "hairpin-veth", + "protectKernelDefaults": true, + "readOnlyPort": 0, + "logging": { + "verbosity": 2 + }, + "serializeImagePulls": false, + "serverTLSBootstrap": true, + "tlsCipherSuites": [ + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_RSA_WITH_AES_256_GCM_SHA384" + ], + "clusterDNS": [ + "10.100.0.10" + ], + "maxPods": 58, + "evictionHard": { + "memory.available": "100Mi", + "nodefs.available": "10%", + "nodefs.inodesFree": "5%" + }, + "kubeReserved": { + "cpu": "70m", + "ephemeral-storage": "1Gi", + "memory": "893Mi" + }, + "systemReserved": { + "cpu": "150m" + }, + "systemReservedCgroup": "/system", + "kubeReservedCgroup": "/runtime", + "providerID": "aws:///us-west-2f/i-1234567890abcdef0" +} diff --git a/nodeadm/test/e2e/cases/mime-userdata-merge/run.sh b/nodeadm/test/e2e/cases/mime-userdata-merge/run.sh new file mode 100755 index 000000000..b09bb8e96 --- /dev/null +++ b/nodeadm/test/e2e/cases/mime-userdata-merge/run.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +source /helpers.sh + +mock::aws +# NOTE: test uses a kubelet version lower than 1.30, since the additional +# config will be written to a drop-in file in 1.30+ +mock::kubelet 1.27.0 +wait::dbus-ready + +nodeadm init --skip run --config-source file://config.yaml + +assert::json-files-equal /etc/kubernetes/kubelet/config.json expected-kubelet-config.json diff --git a/nodeadm/test/e2e/infra/Dockerfile b/nodeadm/test/e2e/infra/Dockerfile index 31a917c31..3983893ec 100644 --- a/nodeadm/test/e2e/infra/Dockerfile +++ b/nodeadm/test/e2e/infra/Dockerfile @@ -34,8 +34,6 @@ COPY --from=imds-mock-build /imds-mock /usr/local/bin/imds-mock # certificateAuthority: Y2VydGlmaWNhdGVBdXRob3JpdHk= # cidr: 10.100.0.0/16 COPY test/e2e/infra/aemm-default-config.json /etc/aemm-default-config.json -COPY test/e2e/infra/aemm-inf1-config.json /etc/aemm-inf1-config.json -COPY test/e2e/infra/aemm-g5-config.json /etc/aemm-g5-config.json COPY test/e2e/infra/nvidia-ctk /usr/bin/nvidia-ctk COPY --from=nodeadm-build /nodeadm /usr/local/bin/nodeadm COPY test/e2e/infra/systemd/kubelet.service /usr/lib/systemd/system/kubelet.service