From c99895053933d1858a84d156e06e8151e4fb759d Mon Sep 17 00:00:00 2001 From: Vitor Savian Date: Wed, 21 Feb 2024 18:55:07 -0300 Subject: [PATCH] Add kine support Signed-off-by: Vitor Savian --- pkg/cli/cmds/server.go | 8 +-- pkg/podexecutor/staticpod.go | 125 ++++++++++++++++++++++------------- pkg/rke2/rke2.go | 6 +- pkg/rke2/rke2_linux.go | 25 ++++--- pkg/staticpod/staticpod.go | 33 ++++++--- 5 files changed, 129 insertions(+), 68 deletions(-) diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index d9996109b9..1bf5d744fa 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -73,10 +73,10 @@ var ( "kube-controller-manager-arg": copyFlag, "kube-cloud-controller-manager-arg": copyFlag, "kube-cloud-controller-arg": dropFlag, // deprecated version of kube-cloud-controller-manager-arg - "datastore-endpoint": dropFlag, - "datastore-cafile": dropFlag, - "datastore-certfile": dropFlag, - "datastore-keyfile": dropFlag, + "datastore-endpoint": copyFlag, + "datastore-cafile": copyFlag, + "datastore-certfile": copyFlag, + "datastore-keyfile": copyFlag, "default-local-storage-path": dropFlag, "disable": { Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: " + strings.Join(DisableItems, ", ") + ")", diff --git a/pkg/podexecutor/staticpod.go b/pkg/podexecutor/staticpod.go index 9d8d9ab203..ad13755ddf 100644 --- a/pkg/podexecutor/staticpod.go +++ b/pkg/podexecutor/staticpod.go @@ -109,18 +109,19 @@ type StaticPodConfig struct { ControlPlaneProbeConfs ControlPlaneEnv ControlPlaneMounts - ManifestsDir string - ImagesDir string - Resolver *images.Resolver - CloudProvider *CloudProviderConfig - DataDir string - AuditPolicyFile string - PSAConfigFile string - KubeletPath string - RuntimeEndpoint string - CISMode bool - DisableETCD bool - IsServer bool + ManifestsDir string + ImagesDir string + Resolver *images.Resolver + CloudProvider *CloudProviderConfig + DataDir string + AuditPolicyFile string + PSAConfigFile string + KubeletPath string + RuntimeEndpoint string + CISMode bool + DisableETCD bool + ExternalDatabase bool + IsServer bool stopKubelet context.CancelFunc } @@ -265,6 +266,11 @@ func (s *StaticPodConfig) APIServer(_ context.Context, etcdReady <-chan struct{} case "--advertise-port", "--basic-auth-file": // This is an option k3s adds that does not exist upstream args = append(args[:i], args[i+1:]...) + case "--etcd-servers": + if s.ExternalDatabase { + args = append(args[:i], args[i+1:]...) + args = append([]string{"--etcd-servers=" + "unixs://" + filepath.Join(s.DataDir, "server", "kine.sock")}, args...) + } case "--audit-log-path": auditLogFile = value case "--kubelet-preferred-address-types": @@ -310,6 +316,12 @@ func (s *StaticPodConfig) APIServer(_ context.Context, etcdReady <-chan struct{} if !s.DisableETCD { files = append(files, etcdNameFile(s.DataDir)) } + + sockets := []string{} + if s.ExternalDatabase { + sockets = append(sockets, kineSock(s.DataDir)) + } + dirs := onlyExisting(ssldirs) if auditLogFile != "" && auditLogFile != "-" { dirs = append(dirs, filepath.Dir(auditLogFile)) @@ -320,41 +332,44 @@ func (s *StaticPodConfig) APIServer(_ context.Context, etcdReady <-chan struct{} dirs = append(dirs, filepath.Join(s.DataDir, "server/cred")) excludeFiles = append(excludeFiles, filepath.Join(s.DataDir, "server/cred/encryption-config.json")) + apiServerArgs := staticpod.Args{ + Command: "kube-apiserver", + Args: args, + Image: image, + Dirs: dirs, + CISMode: s.CISMode, + CPURequest: s.ControlPlaneResources.KubeAPIServerCPURequest, + CPULimit: s.ControlPlaneResources.KubeAPIServerCPULimit, + MemoryRequest: s.ControlPlaneResources.KubeAPIServerMemoryRequest, + MemoryLimit: s.ControlPlaneResources.KubeAPIServerMemoryLimit, + ExtraEnv: s.ControlPlaneEnv.KubeAPIServer, + ExtraMounts: s.ControlPlaneMounts.KubeAPIServer, + ProbeConfs: s.ControlPlaneProbeConfs.KubeAPIServer, + Sockets: sockets, + Files: files, + ExcludeFiles: excludeFiles, + HealthExec: []string{ + "kubectl", + "get", + "--server=https://localhost:6443/", + "--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt", + "--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key", + "--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt", + "--raw=/livez", + }, + ReadyExec: []string{ + "kubectl", + "get", + "--server=https://localhost:6443/", + "--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt", + "--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key", + "--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt", + "--raw=/readyz", + }, + } + return after(etcdReady, func() error { - return staticpod.Run(s.ManifestsDir, staticpod.Args{ - Command: "kube-apiserver", - Args: args, - Image: image, - Dirs: dirs, - CISMode: s.CISMode, - CPURequest: s.ControlPlaneResources.KubeAPIServerCPURequest, - CPULimit: s.ControlPlaneResources.KubeAPIServerCPULimit, - MemoryRequest: s.ControlPlaneResources.KubeAPIServerMemoryRequest, - MemoryLimit: s.ControlPlaneResources.KubeAPIServerMemoryLimit, - ExtraEnv: s.ControlPlaneEnv.KubeAPIServer, - ExtraMounts: s.ControlPlaneMounts.KubeAPIServer, - ProbeConfs: s.ControlPlaneProbeConfs.KubeAPIServer, - Files: files, - ExcludeFiles: excludeFiles, - HealthExec: []string{ - "kubectl", - "get", - "--server=https://localhost:6443/", - "--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt", - "--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key", - "--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt", - "--raw=/livez", - }, - ReadyExec: []string{ - "kubectl", - "get", - "--server=https://localhost:6443/", - "--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt", - "--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key", - "--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt", - "--raw=/readyz", - }, - }) + return staticpod.Run(s.ManifestsDir, apiServerArgs) }) } @@ -373,6 +388,12 @@ func (s *StaticPodConfig) Scheduler(_ context.Context, apiReady <-chan struct{}, if !s.DisableETCD { files = append(files, etcdNameFile(s.DataDir)) } + + sockets := []string{} + if s.ExternalDatabase { + sockets = append(sockets, kineSock(s.DataDir)) + } + args = append(permitPortSharingFlag, args...) return after(apiReady, func() error { return staticpod.Run(s.ManifestsDir, staticpod.Args{ @@ -390,6 +411,7 @@ func (s *StaticPodConfig) Scheduler(_ context.Context, apiReady <-chan struct{}, ExtraMounts: s.ControlPlaneMounts.KubeScheduler, ProbeConfs: s.ControlPlaneProbeConfs.KubeScheduler, Files: files, + Sockets: sockets, }) }) } @@ -438,6 +460,12 @@ func (s *StaticPodConfig) ControllerManager(_ context.Context, apiReady <-chan s if !s.DisableETCD { files = append(files, etcdNameFile(s.DataDir)) } + + sockets := []string{} + if s.ExternalDatabase { + sockets = append(sockets, kineSock(s.DataDir)) + } + return after(apiReady, func() error { extraArgs := []string{ "--flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins", @@ -460,6 +488,7 @@ func (s *StaticPodConfig) ControllerManager(_ context.Context, apiReady <-chan s ExtraMounts: s.ControlPlaneMounts.KubeControllerManager, ProbeConfs: s.ControlPlaneProbeConfs.KubeControllerManager, Files: files, + Sockets: sockets, }) }) } @@ -676,6 +705,10 @@ func chownr(path string, uid, gid int) error { }) } +func kineSock(dataDir string) string { + return filepath.Join(dataDir, "server", "kine.sock") +} + func etcdNameFile(dataDir string) string { return filepath.Join(dataDir, "server", "db", "etcd", "name") } diff --git a/pkg/rke2/rke2.go b/pkg/rke2/rke2.go index 35ac9e514d..08e8303b1b 100644 --- a/pkg/rke2/rke2.go +++ b/pkg/rke2/rke2.go @@ -154,17 +154,18 @@ func setup(clx *cli.Context, cfg Config, isServer bool) error { } // check for missing db name file on a server running etcd, indicating we're rejoining after cluster reset on a different node - if _, err := os.Stat(etcdNameFile(dataDir)); err != nil && os.IsNotExist(err) && isServer && !clx.Bool("disable-etcd") { + if _, err := os.Stat(etcdNameFile(dataDir)); err != nil && os.IsNotExist(err) && isServer && !clx.Bool("disable-etcd") && !clx.IsSet("datastore-endpoint") { clusterReset = true } disabledItems := map[string]bool{ "cloud-controller-manager": !isServer || forceRestart || clx.Bool("disable-cloud-controller"), - "etcd": !isServer || forceRestart || clx.Bool("disable-etcd"), + "etcd": !isServer || forceRestart || clx.Bool("disable-etcd") || clx.IsSet("datastore-endpoint"), "kube-apiserver": !isServer || forceRestart || clx.Bool("disable-apiserver"), "kube-controller-manager": !isServer || forceRestart || clx.Bool("disable-controller-manager"), "kube-scheduler": !isServer || forceRestart || clx.Bool("disable-scheduler"), } + // adding force restart file when cluster reset restore path is passed if clusterResetRestorePath != "" { forceRestartFile := ForceRestartFile(dataDir) @@ -175,6 +176,7 @@ func setup(clx *cli.Context, cfg Config, isServer bool) error { return err } } + return removeDisabledPods(dataDir, containerRuntimeEndpoint, disabledItems, clusterReset) } diff --git a/pkg/rke2/rke2_linux.go b/pkg/rke2/rke2_linux.go index e2889e1a82..833d721264 100644 --- a/pkg/rke2/rke2_linux.go +++ b/pkg/rke2/rke2_linux.go @@ -57,10 +57,17 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta return nil, err } - agentManifestsDir := filepath.Join(dataDir, "agent", config.DefaultPodManifestPath) - agentImagesDir := filepath.Join(dataDir, "agent", "images") + // Verify if the user are using a external database + // and then remove the etcd from the static pod + ExternalDatabase := false + if cmds.ServerConfig.DatastoreEndpoint != "" { + cmds.ServerConfig.ClusterInit = false + ExternalDatabase = true + } managed.RegisterDriver(&etcd.ETCD{}) + agentManifestsDir := filepath.Join(dataDir, "agent", config.DefaultPodManifestPath) + agentImagesDir := filepath.Join(dataDir, "agent", "images") if clx.IsSet("cloud-provider-config") || clx.IsSet("cloud-provider-name") { if clx.IsSet("node-external-ip") { @@ -115,6 +122,7 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta if err != nil { return nil, err } + // Adding PSAs podSecurityConfigFile := clx.String("pod-security-admission-config-file") if podSecurityConfigFile == "" { @@ -141,6 +149,7 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta KubeletPath: cfg.KubeletPath, RuntimeEndpoint: containerRuntimeEndpoint, DisableETCD: clx.Bool("disable-etcd"), + ExternalDatabase: ExternalDatabase, IsServer: isServer, ControlPlaneResources: *controlPlaneResources, ControlPlaneProbeConfs: *controlPlaneProbeConfs, @@ -154,7 +163,7 @@ func parseControlPlaneResources(cfg Config) (*podexecutor.ControlPlaneResources, // resources is a map of the component (kube-apiserver, kube-controller-manager, etc.) to a map[string]*string, // where the key of the downstream map is the `cpu-request`, `cpu-limit`, `memory-request`, or `memory-limit` and // the value corresponds to a pointer to the component resources array - var resources = map[string]map[string]*string{ + resources := map[string]map[string]*string{ KubeAPIServer: { CPURequest: &controlPlaneResources.KubeAPIServerCPURequest, CPULimit: &controlPlaneResources.KubeAPIServerCPULimit, @@ -194,7 +203,7 @@ func parseControlPlaneResources(cfg Config) (*podexecutor.ControlPlaneResources, } // defaultResources contains a map of default resources for each component, used if not explicitly configured. - var defaultResources = map[string]map[string]string{ + defaultResources := map[string]map[string]string{ KubeAPIServer: { CPURequest: "250m", MemoryRequest: "1024Mi", @@ -221,7 +230,7 @@ func parseControlPlaneResources(cfg Config) (*podexecutor.ControlPlaneResources, }, } - var parsedRequestsLimits = make(map[string]string) + parsedRequestsLimits := make(map[string]string) for _, requests := range cfg.ControlPlaneResourceRequests { for _, rawRequest := range strings.Split(requests, ",") { @@ -261,7 +270,7 @@ func parseControlPlaneProbeConfs(cfg Config) (*podexecutor.ControlPlaneProbeConf var controlPlaneProbes podexecutor.ControlPlaneProbeConfs // probes is a map of the component (kube-apiserver, kube-controller-manager, etc.) probe type, and setting, where // the value corresponds to a pointer to the component probes array. - var probes = map[string]map[string]map[string]*int32{ + probes := map[string]map[string]map[string]*int32{ KubeAPIServer: { Liveness: { InitialDelaySeconds: &controlPlaneProbes.KubeAPIServer.Liveness.InitialDelaySeconds, @@ -385,7 +394,7 @@ func parseControlPlaneProbeConfs(cfg Config) (*podexecutor.ControlPlaneProbeConf } // defaultProbeConf contains a map of default probe settings for each type, used if not explicitly configured. - var defaultProbeConf = map[string]map[string]int32{ + defaultProbeConf := map[string]map[string]int32{ // https://github.com/kubernetes/kubernetes/blob/v1.24.0/cmd/kubeadm/app/util/staticpod/utils.go#L246 Liveness: { InitialDelaySeconds: 10, @@ -409,7 +418,7 @@ func parseControlPlaneProbeConfs(cfg Config) (*podexecutor.ControlPlaneProbeConf }, } - var parsedProbeConf = make(map[string]int32) + parsedProbeConf := make(map[string]int32) for _, conf := range cfg.ControlPlaneProbeConf { for _, rawConf := range strings.Split(conf, ",") { diff --git a/pkg/staticpod/staticpod.go b/pkg/staticpod/staticpod.go index 5ff901eb94..d09ff29500 100644 --- a/pkg/staticpod/staticpod.go +++ b/pkg/staticpod/staticpod.go @@ -28,8 +28,13 @@ import ( "sigs.k8s.io/yaml" ) +type typeVolume string + const ( - extraMountPrefix = "extra-mount" + extraMountPrefix = "extra-mount" + socket typeVolume = "socket" + dir typeVolume = "dir" + file typeVolume = "file" ) type ProbeConf struct { @@ -51,6 +56,7 @@ type Args struct { Image name.Reference Dirs []string Files []string + Sockets []string CISMode bool // CIS requires that the manifest be saved with 600 permissions ExcludeFiles []string HealthExec []string @@ -280,8 +286,9 @@ func pod(args Args) (*v1.Pod, error) { } } - addVolumes(p, args.Dirs, true) - addVolumes(p, args.Files, false) + addVolumes(p, args.Sockets, socket) + addVolumes(p, args.Dirs, dir) + addVolumes(p, args.Files, file) addExtraMounts(p, args.ExtraMounts) addExtraEnv(p, args.ExtraEnv) @@ -289,13 +296,23 @@ func pod(args Args) (*v1.Pod, error) { return p, nil } -func addVolumes(p *v1.Pod, src []string, dir bool) { +func addVolumes(p *v1.Pod, src []string, volume typeVolume) { var ( - prefix = "dir" - sourceType = v1.HostPathDirectoryOrCreate - readOnly = false + prefix string + sourceType v1.HostPathType + readOnly bool ) - if !dir { + + switch volume { + case dir: + prefix = "dir" + sourceType = v1.HostPathDirectoryOrCreate + readOnly = false + case socket: + prefix = "sock" + sourceType = v1.HostPathSocket + readOnly = false + default: prefix = "file" sourceType = v1.HostPathFile readOnly = true