Skip to content

Commit

Permalink
Add kine support
Browse files Browse the repository at this point in the history 
Signed-off-by: Vitor Savian <[email protected]>
  • Loading branch information
@vitorsavian
vitorsavian committed Mar 26, 2024
1 parent 6dbc6ff commit c998950
Show file tree
Hide file tree
Showing 5 changed files with 129 additions and 68 deletions.
8 changes: 4 additions & 4 deletions pkg/cli/cmds/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,10 @@ var (
"kube-controller-manager-arg": copyFlag,
"kube-cloud-controller-manager-arg": copyFlag,
"kube-cloud-controller-arg": dropFlag, // deprecated version of kube-cloud-controller-manager-arg
"datastore-endpoint": dropFlag,
"datastore-cafile": dropFlag,
"datastore-certfile": dropFlag,
"datastore-keyfile": dropFlag,
"datastore-endpoint": copyFlag,
"datastore-cafile": copyFlag,
"datastore-certfile": copyFlag,
"datastore-keyfile": copyFlag,
"default-local-storage-path": dropFlag,
"disable": {
Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: " + strings.Join(DisableItems, ", ") + ")",
Expand Down
125 changes: 79 additions & 46 deletions pkg/podexecutor/staticpod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,18 +109,19 @@ type StaticPodConfig struct {
ControlPlaneProbeConfs
ControlPlaneEnv
ControlPlaneMounts
ManifestsDir string
ImagesDir string
Resolver *images.Resolver
CloudProvider *CloudProviderConfig
DataDir string
AuditPolicyFile string
PSAConfigFile string
KubeletPath string
RuntimeEndpoint string
CISMode bool
DisableETCD bool
IsServer bool
ManifestsDir string
ImagesDir string
Resolver *images.Resolver
CloudProvider *CloudProviderConfig
DataDir string
AuditPolicyFile string
PSAConfigFile string
KubeletPath string
RuntimeEndpoint string
CISMode bool
DisableETCD bool
ExternalDatabase bool
IsServer bool

stopKubelet context.CancelFunc
}
Expand Down Expand Up @@ -265,6 +266,11 @@ func (s *StaticPodConfig) APIServer(_ context.Context, etcdReady <-chan struct{}
case "--advertise-port", "--basic-auth-file":
// This is an option k3s adds that does not exist upstream
args = append(args[:i], args[i+1:]...)
case "--etcd-servers":
if s.ExternalDatabase {
args = append(args[:i], args[i+1:]...)
args = append([]string{"--etcd-servers=" + "unixs://" + filepath.Join(s.DataDir, "server", "kine.sock")}, args...)
}
case "--audit-log-path":
auditLogFile = value
case "--kubelet-preferred-address-types":
Expand Down Expand Up @@ -310,6 +316,12 @@ func (s *StaticPodConfig) APIServer(_ context.Context, etcdReady <-chan struct{}
if !s.DisableETCD {
files = append(files, etcdNameFile(s.DataDir))
}

sockets := []string{}
if s.ExternalDatabase {
sockets = append(sockets, kineSock(s.DataDir))
}

dirs := onlyExisting(ssldirs)
if auditLogFile != "" && auditLogFile != "-" {
dirs = append(dirs, filepath.Dir(auditLogFile))
Expand All @@ -320,41 +332,44 @@ func (s *StaticPodConfig) APIServer(_ context.Context, etcdReady <-chan struct{}
dirs = append(dirs, filepath.Join(s.DataDir, "server/cred"))
excludeFiles = append(excludeFiles, filepath.Join(s.DataDir, "server/cred/encryption-config.json"))

apiServerArgs := staticpod.Args{
Command: "kube-apiserver",
Args: args,
Image: image,
Dirs: dirs,
CISMode: s.CISMode,
CPURequest: s.ControlPlaneResources.KubeAPIServerCPURequest,
CPULimit: s.ControlPlaneResources.KubeAPIServerCPULimit,
MemoryRequest: s.ControlPlaneResources.KubeAPIServerMemoryRequest,
MemoryLimit: s.ControlPlaneResources.KubeAPIServerMemoryLimit,
ExtraEnv: s.ControlPlaneEnv.KubeAPIServer,
ExtraMounts: s.ControlPlaneMounts.KubeAPIServer,
ProbeConfs: s.ControlPlaneProbeConfs.KubeAPIServer,
Sockets: sockets,
Files: files,
ExcludeFiles: excludeFiles,
HealthExec: []string{
"kubectl",
"get",
"--server=https://localhost:6443/",
"--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt",
"--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key",
"--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt",
"--raw=/livez",
},
ReadyExec: []string{
"kubectl",
"get",
"--server=https://localhost:6443/",
"--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt",
"--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key",
"--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt",
"--raw=/readyz",
},
}

return after(etcdReady, func() error {
return staticpod.Run(s.ManifestsDir, staticpod.Args{
Command: "kube-apiserver",
Args: args,
Image: image,
Dirs: dirs,
CISMode: s.CISMode,
CPURequest: s.ControlPlaneResources.KubeAPIServerCPURequest,
CPULimit: s.ControlPlaneResources.KubeAPIServerCPULimit,
MemoryRequest: s.ControlPlaneResources.KubeAPIServerMemoryRequest,
MemoryLimit: s.ControlPlaneResources.KubeAPIServerMemoryLimit,
ExtraEnv: s.ControlPlaneEnv.KubeAPIServer,
ExtraMounts: s.ControlPlaneMounts.KubeAPIServer,
ProbeConfs: s.ControlPlaneProbeConfs.KubeAPIServer,
Files: files,
ExcludeFiles: excludeFiles,
HealthExec: []string{
"kubectl",
"get",
"--server=https://localhost:6443/",
"--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt",
"--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key",
"--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt",
"--raw=/livez",
},
ReadyExec: []string{
"kubectl",
"get",
"--server=https://localhost:6443/",
"--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt",
"--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key",
"--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt",
"--raw=/readyz",
},
})
return staticpod.Run(s.ManifestsDir, apiServerArgs)
})
}

Expand All @@ -373,6 +388,12 @@ func (s *StaticPodConfig) Scheduler(_ context.Context, apiReady <-chan struct{},
if !s.DisableETCD {
files = append(files, etcdNameFile(s.DataDir))
}

sockets := []string{}
if s.ExternalDatabase {
sockets = append(sockets, kineSock(s.DataDir))
}

args = append(permitPortSharingFlag, args...)
return after(apiReady, func() error {
return staticpod.Run(s.ManifestsDir, staticpod.Args{
Expand All @@ -390,6 +411,7 @@ func (s *StaticPodConfig) Scheduler(_ context.Context, apiReady <-chan struct{},
ExtraMounts: s.ControlPlaneMounts.KubeScheduler,
ProbeConfs: s.ControlPlaneProbeConfs.KubeScheduler,
Files: files,
Sockets: sockets,
})
})
}
Expand Down Expand Up @@ -438,6 +460,12 @@ func (s *StaticPodConfig) ControllerManager(_ context.Context, apiReady <-chan s
if !s.DisableETCD {
files = append(files, etcdNameFile(s.DataDir))
}

sockets := []string{}
if s.ExternalDatabase {
sockets = append(sockets, kineSock(s.DataDir))
}

return after(apiReady, func() error {
extraArgs := []string{
"--flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins",
Expand All @@ -460,6 +488,7 @@ func (s *StaticPodConfig) ControllerManager(_ context.Context, apiReady <-chan s
ExtraMounts: s.ControlPlaneMounts.KubeControllerManager,
ProbeConfs: s.ControlPlaneProbeConfs.KubeControllerManager,
Files: files,
Sockets: sockets,
})
})
}
Expand Down Expand Up @@ -676,6 +705,10 @@ func chownr(path string, uid, gid int) error {
})
}

func kineSock(dataDir string) string {
return filepath.Join(dataDir, "server", "kine.sock")
}

func etcdNameFile(dataDir string) string {
return filepath.Join(dataDir, "server", "db", "etcd", "name")
}
Expand Down
6 changes: 4 additions & 2 deletions pkg/rke2/rke2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,17 +154,18 @@ func setup(clx *cli.Context, cfg Config, isServer bool) error {
}

// check for missing db name file on a server running etcd, indicating we're rejoining after cluster reset on a different node
if _, err := os.Stat(etcdNameFile(dataDir)); err != nil && os.IsNotExist(err) && isServer && !clx.Bool("disable-etcd") {
if _, err := os.Stat(etcdNameFile(dataDir)); err != nil && os.IsNotExist(err) && isServer && !clx.Bool("disable-etcd") && !clx.IsSet("datastore-endpoint") {
clusterReset = true
}

disabledItems := map[string]bool{
"cloud-controller-manager": !isServer || forceRestart || clx.Bool("disable-cloud-controller"),
"etcd": !isServer || forceRestart || clx.Bool("disable-etcd"),
"etcd": !isServer || forceRestart || clx.Bool("disable-etcd") || clx.IsSet("datastore-endpoint"),
"kube-apiserver": !isServer || forceRestart || clx.Bool("disable-apiserver"),
"kube-controller-manager": !isServer || forceRestart || clx.Bool("disable-controller-manager"),
"kube-scheduler": !isServer || forceRestart || clx.Bool("disable-scheduler"),
}

// adding force restart file when cluster reset restore path is passed
if clusterResetRestorePath != "" {
forceRestartFile := ForceRestartFile(dataDir)
Expand All @@ -175,6 +176,7 @@ func setup(clx *cli.Context, cfg Config, isServer bool) error {
return err
}
}

return removeDisabledPods(dataDir, containerRuntimeEndpoint, disabledItems, clusterReset)
}

Expand Down
25 changes: 17 additions & 8 deletions pkg/rke2/rke2_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,17 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta
return nil, err
}

agentManifestsDir := filepath.Join(dataDir, "agent", config.DefaultPodManifestPath)
agentImagesDir := filepath.Join(dataDir, "agent", "images")
// Verify if the user are using a external database
// and then remove the etcd from the static pod
ExternalDatabase := false
if cmds.ServerConfig.DatastoreEndpoint != "" {
cmds.ServerConfig.ClusterInit = false
ExternalDatabase = true
}

managed.RegisterDriver(&etcd.ETCD{})
agentManifestsDir := filepath.Join(dataDir, "agent", config.DefaultPodManifestPath)
agentImagesDir := filepath.Join(dataDir, "agent", "images")

if clx.IsSet("cloud-provider-config") || clx.IsSet("cloud-provider-name") {
if clx.IsSet("node-external-ip") {
Expand Down Expand Up @@ -115,6 +122,7 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta
if err != nil {
return nil, err
}

// Adding PSAs
podSecurityConfigFile := clx.String("pod-security-admission-config-file")
if podSecurityConfigFile == "" {
Expand All @@ -141,6 +149,7 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta
KubeletPath: cfg.KubeletPath,
RuntimeEndpoint: containerRuntimeEndpoint,
DisableETCD: clx.Bool("disable-etcd"),
ExternalDatabase: ExternalDatabase,
IsServer: isServer,
ControlPlaneResources: *controlPlaneResources,
ControlPlaneProbeConfs: *controlPlaneProbeConfs,
Expand All @@ -154,7 +163,7 @@ func parseControlPlaneResources(cfg Config) (*podexecutor.ControlPlaneResources,
// resources is a map of the component (kube-apiserver, kube-controller-manager, etc.) to a map[string]*string,
// where the key of the downstream map is the `cpu-request`, `cpu-limit`, `memory-request`, or `memory-limit` and
// the value corresponds to a pointer to the component resources array
var resources = map[string]map[string]*string{
resources := map[string]map[string]*string{
KubeAPIServer: {
CPURequest: &controlPlaneResources.KubeAPIServerCPURequest,
CPULimit: &controlPlaneResources.KubeAPIServerCPULimit,
Expand Down Expand Up @@ -194,7 +203,7 @@ func parseControlPlaneResources(cfg Config) (*podexecutor.ControlPlaneResources,
}

// defaultResources contains a map of default resources for each component, used if not explicitly configured.
var defaultResources = map[string]map[string]string{
defaultResources := map[string]map[string]string{
KubeAPIServer: {
CPURequest: "250m",
MemoryRequest: "1024Mi",
Expand All @@ -221,7 +230,7 @@ func parseControlPlaneResources(cfg Config) (*podexecutor.ControlPlaneResources,
},
}

var parsedRequestsLimits = make(map[string]string)
parsedRequestsLimits := make(map[string]string)

for _, requests := range cfg.ControlPlaneResourceRequests {
for _, rawRequest := range strings.Split(requests, ",") {
Expand Down Expand Up @@ -261,7 +270,7 @@ func parseControlPlaneProbeConfs(cfg Config) (*podexecutor.ControlPlaneProbeConf
var controlPlaneProbes podexecutor.ControlPlaneProbeConfs
// probes is a map of the component (kube-apiserver, kube-controller-manager, etc.) probe type, and setting, where
// the value corresponds to a pointer to the component probes array.
var probes = map[string]map[string]map[string]*int32{
probes := map[string]map[string]map[string]*int32{
KubeAPIServer: {
Liveness: {
InitialDelaySeconds: &controlPlaneProbes.KubeAPIServer.Liveness.InitialDelaySeconds,
Expand Down Expand Up @@ -385,7 +394,7 @@ func parseControlPlaneProbeConfs(cfg Config) (*podexecutor.ControlPlaneProbeConf
}

// defaultProbeConf contains a map of default probe settings for each type, used if not explicitly configured.
var defaultProbeConf = map[string]map[string]int32{
defaultProbeConf := map[string]map[string]int32{
// https://github.com/kubernetes/kubernetes/blob/v1.24.0/cmd/kubeadm/app/util/staticpod/utils.go#L246
Liveness: {
InitialDelaySeconds: 10,
Expand All @@ -409,7 +418,7 @@ func parseControlPlaneProbeConfs(cfg Config) (*podexecutor.ControlPlaneProbeConf
},
}

var parsedProbeConf = make(map[string]int32)
parsedProbeConf := make(map[string]int32)

for _, conf := range cfg.ControlPlaneProbeConf {
for _, rawConf := range strings.Split(conf, ",") {
Expand Down
33 changes: 25 additions & 8 deletions pkg/staticpod/staticpod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,13 @@ import (
"sigs.k8s.io/yaml"
)

type typeVolume string

const (
extraMountPrefix = "extra-mount"
extraMountPrefix = "extra-mount"
socket typeVolume = "socket"
dir typeVolume = "dir"
file typeVolume = "file"
)

type ProbeConf struct {
Expand All @@ -51,6 +56,7 @@ type Args struct {
Image name.Reference
Dirs []string
Files []string
Sockets []string
CISMode bool // CIS requires that the manifest be saved with 600 permissions
ExcludeFiles []string
HealthExec []string
Expand Down Expand Up @@ -280,22 +286,33 @@ func pod(args Args) (*v1.Pod, error) {
}
}

addVolumes(p, args.Dirs, true)
addVolumes(p, args.Files, false)
addVolumes(p, args.Sockets, socket)
addVolumes(p, args.Dirs, dir)
addVolumes(p, args.Files, file)

addExtraMounts(p, args.ExtraMounts)
addExtraEnv(p, args.ExtraEnv)

return p, nil
}

func addVolumes(p *v1.Pod, src []string, dir bool) {
func addVolumes(p *v1.Pod, src []string, volume typeVolume) {
var (
prefix = "dir"
sourceType = v1.HostPathDirectoryOrCreate
readOnly = false
prefix string
sourceType v1.HostPathType
readOnly bool
)
if !dir {

switch volume {
case dir:
prefix = "dir"
sourceType = v1.HostPathDirectoryOrCreate
readOnly = false
case socket:
prefix = "sock"
sourceType = v1.HostPathSocket
readOnly = false
default:
prefix = "file"
sourceType = v1.HostPathFile
readOnly = true
Expand Down

0 comments on commit c998950

Please sign in to comment.