diff --git a/cmd/cert/main.go b/cmd/cert/main.go index ff1bdac2..db4c5c0f 100644 --- a/cmd/cert/main.go +++ b/cmd/cert/main.go @@ -5,17 +5,18 @@ import ( "errors" "os" - "github.com/sirupsen/logrus" - "github.com/urfave/cli" "github.com/xiaods/k8e/pkg/cli/cert" "github.com/xiaods/k8e/pkg/cli/cmds" "github.com/xiaods/k8e/pkg/configfilearg" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" ) func main() { app := cmds.NewApp() app.Commands = []cli.Command{ cmds.NewCertCommands( + cert.Check, cert.Rotate, cert.RotateCA, ), @@ -24,4 +25,4 @@ func main() { if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { logrus.Fatal(err) } -} +} \ No newline at end of file diff --git a/cmd/k8e/main.go b/cmd/k8e/main.go index 7502e560..8c14f756 100644 --- a/cmd/k8e/main.go +++ b/cmd/k8e/main.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "io" + "io/fs" "os" "os/exec" "path/filepath" @@ -11,7 +12,7 @@ import ( "strings" "github.com/pkg/errors" - "github.com/rancher/wrangler/pkg/resolvehome" + "github.com/rancher/wrangler/v3/pkg/resolvehome" "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/urfave/cli" @@ -29,6 +30,10 @@ var criDefaultConfigPath = "/etc/crictl.yaml" // main entrypoint for the k8e multicall binary func main() { + if findDebug(os.Args) { + logrus.SetLevel(logrus.DebugLevel) + } + dataDir := findDataDir(os.Args) // Handle direct invocation via symlink alias (multicall binary behavior) @@ -51,7 +56,6 @@ func main() { cmds.NewCRICTL(externalCLIAction("crictl", dataDir)), cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)), cmds.NewCheckConfigCommand(externalCLIAction("check-config", dataDir)), - cmds.NewInitOSConfigCommand(externalCLIAction("init-os-config", dataDir)), cmds.NewTokenCommands( tokenCommand, tokenCommand, @@ -59,25 +63,25 @@ func main() { tokenCommand, tokenCommand, ), - cmds.NewEtcdSnapshotCommand(etcdsnapshotCommand, - cmds.NewEtcdSnapshotSubcommands( - etcdsnapshotCommand, - etcdsnapshotCommand, - etcdsnapshotCommand, - etcdsnapshotCommand), + cmds.NewEtcdSnapshotCommands( + etcdsnapshotCommand, + etcdsnapshotCommand, + etcdsnapshotCommand, + etcdsnapshotCommand, ), - cmds.NewSecretsEncryptCommand(secretsencryptCommand, - cmds.NewSecretsEncryptSubcommands( - secretsencryptCommand, - secretsencryptCommand, - secretsencryptCommand, - secretsencryptCommand, - secretsencryptCommand, - secretsencryptCommand), + cmds.NewSecretsEncryptCommands( + secretsencryptCommand, + secretsencryptCommand, + secretsencryptCommand, + secretsencryptCommand, + secretsencryptCommand, + secretsencryptCommand, + secretsencryptCommand, ), cmds.NewCertCommands( certCommand, certCommand, + certCommand, ), cmds.NewCompletionCommand(internalCLIAction(version.Program+"-completion", dataDir, os.Args)), } @@ -87,11 +91,32 @@ func main() { } } -// findDataDir reads data-dir settings from the CLI args and config file. +// findDebug reads debug settings from the environment, CLI args, and config file. +func findDebug(args []string) bool { + debug, _ := strconv.ParseBool(os.Getenv(version.ProgramUpper + "_DEBUG")) + if debug { + return debug + } + fs := pflag.NewFlagSet("debug-set", pflag.ContinueOnError) + fs.ParseErrorsWhitelist.UnknownFlags = true + fs.SetOutput(io.Discard) + fs.BoolVarP(&debug, "debug", "", false, "(logging) Turn on debug logs") + fs.Parse(args) + if debug { + return debug + } + debug, _ = strconv.ParseBool(configfilearg.MustFindString(args, "debug")) + return debug +} + +// findDataDir reads data-dir settings from the environment, CLI args, and config file. // If not found, the default will be used, which varies depending on whether // k8e is being run as root or not. func findDataDir(args []string) string { - var dataDir string + dataDir := os.Getenv(version.ProgramUpper + "_DATA_DIR") + if dataDir != "" { + return dataDir + } fs := pflag.NewFlagSet("data-dir-set", pflag.ContinueOnError) fs.ParseErrorsWhitelist.UnknownFlags = true fs.SetOutput(io.Discard) @@ -161,7 +186,7 @@ func externalCLI(cli, dataDir string, args []string) error { return stageAndRun(dataDir, cli, append([]string{cli}, args...), false) } -// internalCLIAction returns a function that will call a k8e internal command, be used as the Action of a cli.Command. +// internalCLIAction returns a function that will call a K8e internal command, be used as the Action of a cli.Command. func internalCLIAction(cmd, dataDir string, args []string) func(ctx *cli.Context) error { return func(ctx *cli.Context) error { // We don't want the Info logs seen when printing the autocomplete script @@ -185,16 +210,24 @@ func stageAndRun(dataDir, cmd string, args []string, calledAsInternal bool) erro } logrus.Debugf("Asset dir %s", dir) - var pathEnv string + pathList := []string{ + filepath.Clean(filepath.Join(dir, "..", "cni")), + filepath.Join(dir, "bin"), + } if findPreferBundledBin(args) { - pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux") + string(os.PathListSeparator) + os.Getenv("PATH") + pathList = append( + pathList, + filepath.Join(dir, "bin", "aux"), + os.Getenv("PATH"), + ) } else { - pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + os.Getenv("PATH") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux") - } - if err := os.Setenv("PATH", pathEnv); err != nil { - return err + pathList = append( + pathList, + os.Getenv("PATH"), + filepath.Join(dir, "bin", "aux"), + ) } - if err := os.Setenv(version.ProgramUpper+"_DATA_DIR", dir); err != nil { + if err := os.Setenv("PATH", strings.Join(pathList, string(os.PathListSeparator))); err != nil { return err } @@ -269,6 +302,53 @@ func extract(dataDir string) (string, error) { return "", err } + // Create a stable CNI bin dir and place it first in the path so that users have a + // consistent location to drop their own CNI plugin binaries. + cniPath := filepath.Join(dataDir, "data", "cni") + cniBin := filepath.Join(dir, "bin", "cni") + if err := os.MkdirAll(cniPath, 0755); err != nil { + return "", err + } + // Create symlink that points at the cni multicall binary itself + logrus.Debugf("Creating symlink %s -> %s", filepath.Join(cniPath, "cni"), cniBin) + os.Remove(filepath.Join(cniPath, "cni")) + if err := os.Symlink(cniBin, filepath.Join(cniPath, "cni")); err != nil { + return "", err + } + + // Find symlinks that point to the cni multicall binary, and clone them in the stable CNI bin dir. + // Non-symlink plugins in the stable CNI bin dir will not be overwritten, to allow users to replace our + // CNI plugins with their own versions if they want. Note that the cni multicall binary itself is always + // symlinked into the stable bin dir and should not be replaced. + ents, err := os.ReadDir(filepath.Join(tempDest, "bin")) + if err != nil { + return "", err + } + for _, ent := range ents { + if info, err := ent.Info(); err == nil && info.Mode()&fs.ModeSymlink != 0 { + if target, err := os.Readlink(filepath.Join(tempDest, "bin", ent.Name())); err == nil && target == "cni" { + src := filepath.Join(cniPath, ent.Name()) + // Check if plugin already exists in stable CNI bin dir + if info, err := os.Lstat(src); err == nil { + if info.Mode()&fs.ModeSymlink != 0 { + // Exists and is a symlink, remove it so we can create a new symlink for the new bin. + os.Remove(src) + } else { + // Not a symlink, leave it alone + logrus.Debugf("Not replacing non-symlink CNI plugin %s with mode %O", src, info.Mode()) + continue + } + } + logrus.Debugf("Creating symlink %s -> %s", src, cniBin) + if err := os.Symlink(cniBin, src); err != nil { + return "", err + } + } + } + } + + // Rotate 'current' symlink into 'previous', and create a new 'current' that points + // at the new directory. currentSymLink := filepath.Join(dataDir, "data", "current") previousSymLink := filepath.Join(dataDir, "data", "previous") if _, err := os.Lstat(currentSymLink); err == nil { @@ -279,7 +359,14 @@ func extract(dataDir string) (string, error) { if err := os.Symlink(dir, currentSymLink); err != nil { return "", err } - return dir, os.Rename(tempDest, dir) + + // Rename the new directory into place after updating symlinks, so that the k8e binary check at the start + // of this function only succeeds if everything else has been completed successfully. + if err := os.Rename(tempDest, dir); err != nil { + return "", err + } + + return dir, nil } // findCriConfig returns the path to crictl.yaml diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index 01026813..f798f230 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -4,8 +4,8 @@ import ( "os" "path/filepath" - "github.com/xiaods/k8e/pkg/version" "github.com/urfave/cli" + "github.com/xiaods/k8e/pkg/version" ) type Agent struct { diff --git a/pkg/cli/cmds/certs.go b/pkg/cli/cmds/certs.go index bd0133e7..3c00a5d1 100644 --- a/pkg/cli/cmds/certs.go +++ b/pkg/cli/cmds/certs.go @@ -23,11 +23,12 @@ var ( DataDirFlag, &cli.StringSliceFlag{ Name: "service,s", - Usage: "List of services to rotate certificates for. Options include (admin, api-server, controller-manager, scheduler, " + version.Program + "-controller, " + version.Program + "-server, cloud-controller, etcd, auth-proxy, kubelet)", + Usage: "List of services to manage certificates for. Options include (admin, api-server, controller-manager, scheduler, supervisor, " + version.Program + "-controller, " + version.Program + "-server, cloud-controller, etcd, auth-proxy, kubelet, kube-proxy)", Value: &ServicesList, }, } CertRotateCACommandFlags = []cli.Flag{ + DataDirFlag, cli.StringFlag{ Name: "server,s", Usage: "(cluster) Server to connect to", @@ -35,11 +36,6 @@ var ( Value: "https://127.0.0.1:6443", Destination: &ServerConfig.ServerURL, }, - cli.StringFlag{ - Name: "data-dir,d", - Usage: "(data) Folder to hold state default /var/lib/" + version.Program + " or ${HOME}/." + version.Program + " if not root", - Destination: &ServerConfig.DataDir, - }, cli.StringFlag{ Name: "path", Usage: "Path to directory containing new CA certificates", @@ -54,13 +50,21 @@ var ( } ) -func NewCertCommands(rotate, rotateCA func(ctx *cli.Context) error) cli.Command { +func NewCertCommands(check, rotate, rotateCA func(ctx *cli.Context) error) cli.Command { return cli.Command{ Name: CertCommand, Usage: "Manage K8e certificates", SkipFlagParsing: false, SkipArgReorder: true, Subcommands: []cli.Command{ + { + Name: "check", + Usage: "Check " + version.Program + " component certificates on disk", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: check, + Flags: CertRotateCommandFlags, + }, { Name: "rotate", Usage: "Rotate " + version.Program + " component certificates on disk", diff --git a/pkg/cli/cmds/etcd_snapshot.go b/pkg/cli/cmds/etcd_snapshot.go index 49b34d95..70026484 100644 --- a/pkg/cli/cmds/etcd_snapshot.go +++ b/pkg/cli/cmds/etcd_snapshot.go @@ -21,6 +21,17 @@ var EtcdSnapshotFlags = []cli.Flag{ Destination: &AgentConfig.NodeName, }, DataDirFlag, + &cli.StringFlag{ + Name: "etcd-token,t", + Usage: "(cluster) Shared secret used to authenticate to etcd server", + Destination: &ServerConfig.Token, + }, + &cli.StringFlag{ + Name: "etcd-server, s", + Usage: "(cluster) Server with etcd role to connect to for snapshot management operations", + Value: "https://127.0.0.1:6443", + Destination: &ServerConfig.ServerURL, + }, &cli.StringFlag{ Name: "dir,etcd-snapshot-dir", Usage: "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)", @@ -37,6 +48,12 @@ var EtcdSnapshotFlags = []cli.Flag{ Usage: "(db) Compress etcd snapshot", Destination: &ServerConfig.EtcdSnapshotCompress, }, + &cli.IntFlag{ + Name: "snapshot-retention,etcd-snapshot-retention", + Usage: "(db) Number of snapshots to retain.", + Destination: &ServerConfig.EtcdSnapshotRetention, + Value: defaultSnapshotRentention, + }, &cli.BoolFlag{ Name: "s3,etcd-s3", Usage: "(db) Enable backup to S3", @@ -86,6 +103,16 @@ var EtcdSnapshotFlags = []cli.Flag{ Usage: "(db) S3 folder", Destination: &ServerConfig.EtcdS3Folder, }, + &cli.StringFlag{ + Name: "s3-proxy,etcd-s3-proxy", + Usage: "(db) Proxy server to use when connecting to S3, overriding any proxy-releated environment variables", + Destination: &ServerConfig.EtcdS3Proxy, + }, + &cli.StringFlag{ + Name: "s3-config-secret,etcd-s3-config-secret", + Usage: "(db) Name of secret in the kube-system namespace used to configure S3, if etcd-s3 is enabled and no other etcd-s3 options are set", + Destination: &ServerConfig.EtcdS3ConfigSecret, + }, &cli.BoolFlag{ Name: "s3-insecure,etcd-s3-insecure", Usage: "(db) Disables S3 over HTTPS", @@ -95,65 +122,54 @@ var EtcdSnapshotFlags = []cli.Flag{ Name: "s3-timeout,etcd-s3-timeout", Usage: "(db) S3 timeout", Destination: &ServerConfig.EtcdS3Timeout, - Value: 30 * time.Second, + Value: 5 * time.Minute, }, } -func NewEtcdSnapshotCommand(action func(*cli.Context) error, subcommands []cli.Command) cli.Command { +func NewEtcdSnapshotCommands(delete, list, prune, save func(ctx *cli.Context) error) cli.Command { return cli.Command{ Name: EtcdSnapshotCommand, - Usage: "Trigger an immediate etcd snapshot", SkipFlagParsing: false, SkipArgReorder: true, - Action: action, - Subcommands: subcommands, - Flags: EtcdSnapshotFlags, - } -} - -func NewEtcdSnapshotSubcommands(delete, list, prune, save func(ctx *cli.Context) error) []cli.Command { - return []cli.Command{ - { - Name: "delete", - Usage: "Delete given snapshot(s)", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: delete, - Flags: EtcdSnapshotFlags, - }, - { - Name: "ls", - Aliases: []string{"list", "l"}, - Usage: "List snapshots", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: list, - Flags: append(EtcdSnapshotFlags, &cli.StringFlag{ - Name: "o,output", - Usage: "(db) List format. Default: standard. Optional: json", - Destination: &ServerConfig.EtcdListFormat, - }), - }, - { - Name: "prune", - Usage: "Remove snapshots that match the name prefix that exceed the configured retention count", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: prune, - Flags: append(EtcdSnapshotFlags, &cli.IntFlag{ - Name: "snapshot-retention", - Usage: "(db) Number of snapshots to retain. Default: 5", - Destination: &ServerConfig.EtcdSnapshotRetention, - Value: defaultSnapshotRentention, - }), - }, - { - Name: "save", - Usage: "Trigger an immediate etcd snapshot", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: save, - Flags: EtcdSnapshotFlags, + Subcommands: []cli.Command{ + { + Name: "save", + Usage: "Trigger an immediate etcd snapshot", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: save, + Flags: EtcdSnapshotFlags, + }, + { + Name: "delete", + Usage: "Delete given snapshot(s)", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: delete, + Flags: EtcdSnapshotFlags, + }, + { + Name: "ls", + Aliases: []string{"list", "l"}, + Usage: "List snapshots", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: list, + Flags: append(EtcdSnapshotFlags, &cli.StringFlag{ + Name: "o,output", + Usage: "(db) List format. Default: standard. Optional: json", + Destination: &ServerConfig.EtcdListFormat, + }), + }, + { + Name: "prune", + Usage: "Remove snapshots that match the name prefix that exceed the configured retention count", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: prune, + Flags: EtcdSnapshotFlags, + }, }, + Flags: EtcdSnapshotFlags, } -} +} \ No newline at end of file diff --git a/pkg/cli/cmds/secrets_encrypt.go b/pkg/cli/cmds/secrets_encrypt.go index 6373cc2f..77b7a158 100644 --- a/pkg/cli/cmds/secrets_encrypt.go +++ b/pkg/cli/cmds/secrets_encrypt.go @@ -7,100 +7,90 @@ import ( const SecretsEncryptCommand = "secrets-encrypt" -var EncryptFlags = []cli.Flag{ - DataDirFlag, - ServerToken, - &cli.StringFlag{ - Name: "server, s", - Usage: "(cluster) Server to connect to", - EnvVar: version.ProgramUpper + "_URL", - Value: "https://127.0.0.1:6443", - Destination: &ServerConfig.ServerURL, - }, -} - -func NewSecretsEncryptCommand(action func(*cli.Context) error, subcommands []cli.Command) cli.Command { - return cli.Command{ - Name: SecretsEncryptCommand, - Usage: "Control secrets encryption and keys rotation", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: action, - Subcommands: subcommands, +var ( + forceFlag = &cli.BoolFlag{ + Name: "f,force", + Usage: "Force this stage.", + Destination: &ServerConfig.EncryptForce, } -} - -func NewSecretsEncryptSubcommands(status, enable, disable, prepare, rotate, reencrypt func(ctx *cli.Context) error) []cli.Command { - return []cli.Command{ - { - Name: "status", - Usage: "Print current status of secrets encryption", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: status, - Flags: append(EncryptFlags, &cli.StringFlag{ - Name: "output,o", - Usage: "Status format. Default: text. Optional: json", - Destination: &ServerConfig.EncryptOutput, - }), - }, - { - Name: "enable", - Usage: "Enable secrets encryption", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: enable, - Flags: EncryptFlags, - }, - { - Name: "disable", - Usage: "Disable secrets encryption", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: disable, - Flags: EncryptFlags, + EncryptFlags = []cli.Flag{ + DataDirFlag, + ServerToken, + &cli.StringFlag{ + Name: "server, s", + Usage: "(cluster) Server to connect to", + EnvVar: version.ProgramUpper + "_URL", + Value: "https://127.0.0.1:6443", + Destination: &ServerConfig.ServerURL, }, - { - Name: "prepare", - Usage: "Prepare for encryption keys rotation", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: prepare, - Flags: append(EncryptFlags, &cli.BoolFlag{ - Name: "f,force", - Usage: "Force preparation.", - Destination: &ServerConfig.EncryptForce, - }), - }, - { - Name: "rotate", - Usage: "Rotate secrets encryption keys", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: rotate, - Flags: append(EncryptFlags, &cli.BoolFlag{ - Name: "f,force", - Usage: "Force key rotation.", - Destination: &ServerConfig.EncryptForce, - }), - }, - { - Name: "reencrypt", - Usage: "Reencrypt all data with new encryption key", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: reencrypt, - Flags: append(EncryptFlags, - &cli.BoolFlag{ - Name: "f,force", - Usage: "Force secrets reencryption.", - Destination: &ServerConfig.EncryptForce, - }, - &cli.BoolFlag{ - Name: "skip", - Usage: "Skip removing old key", - Destination: &ServerConfig.EncryptSkip, + } +) + +func NewSecretsEncryptCommands(status, enable, disable, prepare, rotate, reencrypt, rotateKeys func(ctx *cli.Context) error) cli.Command { + return cli.Command{ + Name: SecretsEncryptCommand, + Usage: "Control secrets encryption and keys rotation", + SkipArgReorder: true, + Subcommands: []cli.Command{ + { + Name: "status", + Usage: "Print current status of secrets encryption", + SkipArgReorder: true, + Action: status, + Flags: append(EncryptFlags, &cli.StringFlag{ + Name: "output,o", + Usage: "Status format. Default: text. Optional: json", + Destination: &ServerConfig.EncryptOutput, }), + }, + { + Name: "enable", + Usage: "Enable secrets encryption", + SkipArgReorder: true, + Action: enable, + Flags: EncryptFlags, + }, + { + Name: "disable", + Usage: "Disable secrets encryption", + SkipArgReorder: true, + Action: disable, + Flags: EncryptFlags, + }, + { + Name: "prepare", + Usage: "Prepare for encryption keys rotation", + SkipArgReorder: true, + Action: prepare, + Flags: append(EncryptFlags, forceFlag), + }, + { + Name: "rotate", + Usage: "Rotate secrets encryption keys", + SkipArgReorder: true, + Action: rotate, + Flags: append(EncryptFlags, forceFlag), + }, + { + Name: "reencrypt", + Usage: "Reencrypt all data with new encryption key", + SkipArgReorder: true, + Action: reencrypt, + Flags: append(EncryptFlags, + forceFlag, + &cli.BoolFlag{ + Name: "skip", + Usage: "Skip removing old key", + Destination: &ServerConfig.EncryptSkip, + }), + }, + { + Name: "rotate-keys", + Usage: "(experimental) Dynamically rotates secrets encryption keys and re-encrypt secrets", + SkipArgReorder: true, + Action: rotateKeys, + Flags: EncryptFlags, + }, }, } } diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 2ccb3e11..28e62164 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -5,8 +5,8 @@ import ( "sync" "time" - "github.com/urfave/cli" "github.com/xiaods/k8e/pkg/version" + "github.com/urfave/cli" ) const ( @@ -36,7 +36,7 @@ type Server struct { ClusterDomain string // The port which kubectl clients can access k8s HTTPSPort int - // The port which custom k8e API runs on + // The port which custom k3s API runs on SupervisorPort int // The port which kube-apiserver runs on APIServerPort int @@ -45,11 +45,10 @@ type Server struct { DisableAgent bool KubeConfigOutput string KubeConfigMode string + KubeConfigGroup string HelmJobImage string TLSSan cli.StringSlice TLSSanSecurity bool - BindAddress string - EnablePProf bool ExtraAPIArgs cli.StringSlice ExtraEtcdArgs cli.StringSlice ExtraSchedulerArgs cli.StringSlice @@ -60,6 +59,7 @@ type Server struct { DatastoreCAFile string DatastoreCertFile string DatastoreKeyFile string + KineTLS bool AdvertiseIP string AdvertisePort int DisableScheduler bool @@ -68,9 +68,11 @@ type Server struct { DefaultLocalStoragePath string DisableCCM bool DisableHelmController bool + DisableKubeProxy bool DisableAPIServer bool DisableControllerManager bool DisableETCD bool + EmbeddedRegistry bool ClusterInit bool ClusterReset bool ClusterResetRestorePath string @@ -80,6 +82,7 @@ type Server struct { EncryptSkip bool SystemDefaultRegistry string StartupHooks []StartupHook + SupervisorMetrics bool EtcdSnapshotName string EtcdDisableSnapshots bool EtcdExposeMetrics bool @@ -97,16 +100,20 @@ type Server struct { EtcdS3BucketName string EtcdS3Region string EtcdS3Folder string + EtcdS3Proxy string + EtcdS3ConfigSecret string EtcdS3Timeout time.Duration EtcdS3Insecure bool + ServiceLBNamespace string } var ( ServerConfig Server DataDirFlag = &cli.StringFlag{ Name: "data-dir,d", - Usage: "(data) Folder to hold state default /var/lib/" + version.Program + " or ${HOME}/." + version.Program + " if not root", + Usage: "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root", Destination: &ServerConfig.DataDir, + EnvVar: version.ProgramUpper + "_DATA_DIR", } ServerToken = &cli.StringFlag{ Name: "token,t", @@ -170,11 +177,7 @@ var ServerFlags = []cli.Flag{ VModule, LogFile, AlsoLogToStderr, - &cli.StringFlag{ - Name: "bind-address", - Usage: "(listener) " + version.Program + " bind address (default: 0.0.0.0)", - Destination: &ServerConfig.BindAddress, - }, + BindAddressFlag, &cli.IntFlag{ Name: "https-listen-port", Usage: "(listener) HTTPS listen port", @@ -183,7 +186,7 @@ var ServerFlags = []cli.Flag{ }, &cli.StringFlag{ Name: "advertise-address", - Usage: "(listener) IPv4 address that apiserver uses to advertise to members of the cluster (default: node-external-ip/node-ip)", + Usage: "(listener) IPv4/IPv6 address that apiserver uses to advertise to members of the cluster (default: node-external-ip/node-ip)", Destination: &ServerConfig.AdvertiseIP, }, &cli.IntFlag{ @@ -196,9 +199,9 @@ var ServerFlags = []cli.Flag{ Usage: "(listener) Add additional hostnames or IPv4/IPv6 addresses as Subject Alternative Names on the server TLS cert", Value: &ServerConfig.TLSSan, }, - &cli.BoolFlag{ + &cli.BoolTFlag{ Name: "tls-san-security", - Usage: "(listener) Protect the server TLS cert by refusing to add Subject Alternative Names not associated with the kubernetes apiserver service, server nodes, or values of the tls-san option (default: false)", + Usage: "(listener) Protect the server TLS cert by refusing to add Subject Alternative Names not associated with the kubernetes apiserver service, server nodes, or values of the tls-san option (default: true)", Destination: &ServerConfig.TLSSanSecurity, }, DataDirFlag, @@ -213,6 +216,12 @@ var ServerFlags = []cli.Flag{ Destination: &ServerConfig.EgressSelectorMode, Value: "agent", }, + &cli.StringFlag{ + Name: "servicelb-namespace", + Usage: "(networking) Namespace of the pods for the servicelb component", + Destination: &ServerConfig.ServiceLBNamespace, + Value: "kube-system", + }, &cli.StringFlag{ Name: "write-kubeconfig,o", Usage: "(client) Write kubeconfig for admin client to this file", @@ -225,6 +234,12 @@ var ServerFlags = []cli.Flag{ Destination: &ServerConfig.KubeConfigMode, EnvVar: version.ProgramUpper + "_KUBECONFIG_MODE", }, + &cli.StringFlag{ + Name: "write-kubeconfig-group", + Usage: "(client) Write kubeconfig with this group", + Destination: &ServerConfig.KubeConfigGroup, + EnvVar: version.ProgramUpper + "_KUBECONFIG_GROUP", + }, &cli.StringFlag{ Name: "helm-job-image", Usage: "(helm) Default image to use for helm jobs", @@ -233,7 +248,7 @@ var ServerFlags = []cli.Flag{ ServerToken, &cli.StringFlag{ Name: "token-file", - Usage: "(cluster) File containing the cluster-secret/token", + Usage: "(cluster) File containing the token", Destination: &ServerConfig.TokenFile, EnvVar: version.ProgramUpper + "_TOKEN_FILE", }, @@ -281,9 +296,15 @@ var ServerFlags = []cli.Flag{ Usage: "(flags) Customized flag for kube-cloud-controller-manager process", Value: &ServerConfig.ExtraCloudControllerArgs, }, + &cli.BoolFlag{ + Name: "kine-tls", + Usage: "(experimental/db) Enable TLS on the kine etcd server socket", + Destination: &ServerConfig.KineTLS, + Hidden: true, + }, &cli.StringFlag{ Name: "datastore-endpoint", - Usage: "(db) Specify etcd, Mysql, Postgres, or Sqlite (default) data source name", + Usage: "(db) Specify etcd, NATS, MySQL, Postgres, or SQLite (default) data source name", Destination: &ServerConfig.DatastoreEndpoint, EnvVar: version.ProgramUpper + "_DATASTORE_ENDPOINT", }, @@ -392,6 +413,16 @@ var ServerFlags = []cli.Flag{ Usage: "(db) S3 folder", Destination: &ServerConfig.EtcdS3Folder, }, + &cli.StringFlag{ + Name: "etcd-s3-proxy", + Usage: "(db) Proxy server to use when connecting to S3, overriding any proxy-releated environment variables", + Destination: &ServerConfig.EtcdS3Proxy, + }, + &cli.StringFlag{ + Name: "etcd-s3-config-secret", + Usage: "(db) Name of secret in the kube-system namespace used to configure S3, if etcd-s3 is enabled and no other etcd-s3 options are set", + Destination: &ServerConfig.EtcdS3ConfigSecret, + }, &cli.BoolFlag{ Name: "etcd-s3-insecure", Usage: "(db) Disables S3 over HTTPS", @@ -445,6 +476,16 @@ var ServerFlags = []cli.Flag{ Usage: "(experimental/components) Disable running etcd", Destination: &ServerConfig.DisableETCD, }, + &cli.BoolFlag{ + Name: "embedded-registry", + Usage: "(experimental/components) Enable embedded distributed container registry; requires use of embedded containerd; when enabled agents will also listen on the supervisor port", + Destination: &ServerConfig.EmbeddedRegistry, + }, + &cli.BoolFlag{ + Name: "supervisor-metrics", + Usage: "(experimental/components) Enable serving " + version.Program + " internal metrics on the supervisor port; when enabled agents will also listen on the supervisor port", + Destination: &ServerConfig.SupervisorMetrics, + }, NodeNameFlag, WithNodeIDFlag, NodeLabels, @@ -453,7 +494,9 @@ var ServerFlags = []cli.Flag{ ImageCredProvConfigFlag, DockerFlag, CRIEndpointFlag, + DefaultRuntimeFlag, ImageServiceEndpointFlag, + DisableDefaultRegistryEndpointFlag, PauseImageFlag, SnapshotterFlag, PrivateRegistryFlag, @@ -467,7 +510,13 @@ var ServerFlags = []cli.Flag{ NodeIPFlag, NodeExternalIPFlag, ResolvConfFlag, + FlannelIfaceFlag, + FlannelConfFlag, + FlannelCniConfFileFlag, + VPNAuth, + VPNAuthFile, ExtraKubeletArgs, + ExtraKubeProxyArgs, ProtectKernelDefaultsFlag, &cli.BoolFlag{ Name: "secrets-encryption", @@ -475,11 +524,7 @@ var ServerFlags = []cli.Flag{ Destination: &ServerConfig.EncryptSecrets, }, // Experimental flags - &cli.BoolFlag{ - Name: "enable-pprof", - Usage: "(experimental) Enable pprof endpoint on supervisor port", - Destination: &ServerConfig.EnablePProf, - }, + EnablePProfFlag, &cli.BoolFlag{ Name: "rootless", Usage: "(experimental) Run rootless", @@ -491,19 +536,6 @@ var ServerFlags = []cli.Flag{ // Hidden/Deprecated flags below - DisableSELinuxFlag, - &cli.StringSliceFlag{ - Name: "no-deploy", - Usage: "(deprecated) Do not deploy packaged components (valid items: " + DisableItems + ")", - Hidden: true, - }, - &cli.StringFlag{ - Name: "cluster-secret", - Usage: "(deprecated) use --token", - Destination: &ServerConfig.ClusterSecret, - EnvVar: version.ProgramUpper + "_CLUSTER_SECRET", - Hidden: true, - }, &cli.BoolFlag{ Name: "disable-agent", Usage: "Do not run a local agent and register a local kubelet", @@ -529,8 +561,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Name: "server", Usage: "Run management server", UsageText: appName + " server [OPTIONS]", - Before: CheckSELinuxFlags, Action: action, Flags: ServerFlags, } -} +} \ No newline at end of file diff --git a/pkg/daemons/executor/embed_linux.go b/pkg/daemons/executor/embed_linux.go index 0747de62..a1224581 100644 --- a/pkg/daemons/executor/embed_linux.go +++ b/pkg/daemons/executor/embed_linux.go @@ -4,8 +4,6 @@ package executor import ( - daemonconfig "github.com/xiaods/k8e/pkg/daemons/config" - // registering k8e cloud provider _ "github.com/xiaods/k8e/pkg/cloudprovider" )