From 2645837aa44abd5d32ae9806d9370fb5968ed8fe Mon Sep 17 00:00:00 2001 From: Tommy Xiao Date: Tue, 22 Oct 2024 15:43:35 +0800 Subject: [PATCH] fix: update daemons --- go.mod | 2 +- pkg/agent/templates/templates_linux.go | 6 -- pkg/cli/cmds/server.go | 4 -- pkg/daemons/control/server.go | 60 ++++++++++++++----- pkg/daemons/control/tunnel.go | 56 +++++------------- pkg/daemons/executor/embed_linux.go | 10 ---- pkg/daemons/executor/embed_windows.go | 81 -------------------------- 7 files changed, 64 insertions(+), 155 deletions(-) delete mode 100644 pkg/daemons/executor/embed_linux.go delete mode 100644 pkg/daemons/executor/embed_windows.go diff --git a/go.mod b/go.mod index 6ab93322..2d8b0c21 100644 --- a/go.mod +++ b/go.mod @@ -74,7 +74,6 @@ replace ( ) require ( - github.com/Microsoft/hcsshim v0.12.6 github.com/Mirantis/cri-dockerd v0.0.0-00010101000000-000000000000 github.com/blang/semver/v4 v4.0.0 github.com/containerd/aufs v1.0.0 @@ -183,6 +182,7 @@ require ( github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Microsoft/hcsshim v0.12.6 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/Rican7/retry v0.1.0 // indirect github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e // indirect diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go index 6fb66094..50055fc3 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -62,12 +62,6 @@ enable_keychain = true {{end}} {{end}} -{{- if not .NodeConfig.NoFlannel }} -[plugins."io.containerd.grpc.v1.cri".cni] - bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}" - conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}" -{{end}} - {{- if or .NodeConfig.Containerd.BlockIOConfig .NodeConfig.Containerd.RDTConfig }} [plugins."io.containerd.service.v1.tasks-service"] {{ if .NodeConfig.Containerd.BlockIOConfig }}blockio_config_file = "{{ .NodeConfig.Containerd.BlockIOConfig }}"{{end}} diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 28e62164..b551f40d 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -510,13 +510,9 @@ var ServerFlags = []cli.Flag{ NodeIPFlag, NodeExternalIPFlag, ResolvConfFlag, - FlannelIfaceFlag, - FlannelConfFlag, - FlannelCniConfFileFlag, VPNAuth, VPNAuthFile, ExtraKubeletArgs, - ExtraKubeProxyArgs, ProtectKernelDefaultsFlag, &cli.BoolFlag{ Name: "secrets-encryption", diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 6a639754..08742b32 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -9,8 +9,6 @@ import ( "strings" "time" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" "github.com/xiaods/k8e/pkg/authenticator" "github.com/xiaods/k8e/pkg/cluster" "github.com/xiaods/k8e/pkg/daemons/config" @@ -18,10 +16,13 @@ import ( "github.com/xiaods/k8e/pkg/daemons/executor" "github.com/xiaods/k8e/pkg/util" "github.com/xiaods/k8e/pkg/version" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" authorizationv1 "k8s.io/api/authorization/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" - proxyutil "k8s.io/kubernetes/pkg/proxy/util" + "k8s.io/kubernetes/pkg/registry/core/node" // for client metric registration _ "k8s.io/component-base/metrics/prometheus/restclient" @@ -30,6 +31,7 @@ import ( func Server(ctx context.Context, cfg *config.Control) error { rand.Seed(time.Now().UTC().UnixNano()) + logsapi.ReapplyHandling = logsapi.ReapplyHandlingIgnoreUnchanged if err := prepare(ctx, cfg); err != nil { return errors.Wrap(err, "preparing server") } @@ -40,7 +42,7 @@ func Server(ctx context.Context, cfg *config.Control) error { } cfg.Runtime.Tunnel = tunnel - proxyutil.DisableProxyHostnameCheck = true + node.DisableProxyHostnameCheck = true authArgs := []string{ "--basic-auth-file=" + cfg.Runtime.PasswdFile, @@ -77,7 +79,7 @@ func Server(ctx context.Context, cfg *config.Control) error { } } - if !cfg.DisableCCM { + if !cfg.DisableCCM || !cfg.DisableServiceLB { if err := cloudControllerManager(ctx, cfg); err != nil { return err } @@ -90,7 +92,6 @@ func controllerManager(ctx context.Context, cfg *config.Control) error { runtime := cfg.Runtime argsMap := map[string]string{ "controllers": "*,tokencleaner", - "feature-gates": "JobTrackingWithFinalizers=true", "kubeconfig": runtime.KubeConfigController, "authorization-kubeconfig": runtime.KubeConfigController, "authentication-kubeconfig": runtime.KubeConfigController, @@ -120,6 +121,13 @@ func controllerManager(ctx context.Context, cfg *config.Control) error { argsMap["controllers"] = argsMap["controllers"] + ",-service,-route,-cloud-node-lifecycle" } + if cfg.VLevel != 0 { + argsMap["v"] = strconv.Itoa(cfg.VLevel) + } + if cfg.VModule != "" { + argsMap["vmodule"] = cfg.VModule + } + args := config.GetArgs(argsMap, cfg.ExtraControllerArgs) logrus.Infof("Running kube-controller-manager %s", config.ArgString(args)) @@ -139,6 +147,14 @@ func scheduler(ctx context.Context, cfg *config.Control) error { if cfg.NoLeaderElect { argsMap["leader-elect"] = "false" } + + if cfg.VLevel != 0 { + argsMap["v"] = strconv.Itoa(cfg.VLevel) + } + if cfg.VModule != "" { + argsMap["vmodule"] = cfg.VModule + } + args := config.GetArgs(argsMap, cfg.ExtraSchedulerAPIArgs) logrus.Infof("Running kube-scheduler %s", config.ArgString(args)) @@ -147,9 +163,7 @@ func scheduler(ctx context.Context, cfg *config.Control) error { func apiServer(ctx context.Context, cfg *config.Control) error { runtime := cfg.Runtime - argsMap := map[string]string{ - "feature-gates": "JobTrackingWithFinalizers=true", - } + argsMap := map[string]string{} setupStorageBackend(argsMap, cfg) @@ -185,7 +199,11 @@ func apiServer(ctx context.Context, cfg *config.Control) error { argsMap["kubelet-certificate-authority"] = runtime.ServerCA argsMap["kubelet-client-certificate"] = runtime.ClientKubeAPICert argsMap["kubelet-client-key"] = runtime.ClientKubeAPIKey - argsMap["kubelet-preferred-address-types"] = "InternalIP,ExternalIP,Hostname" + if cfg.FlannelExternalIP { + argsMap["kubelet-preferred-address-types"] = "ExternalIP,InternalIP,Hostname" + } else { + argsMap["kubelet-preferred-address-types"] = "InternalIP,ExternalIP,Hostname" + } argsMap["requestheader-client-ca-file"] = runtime.RequestHeaderCA argsMap["requestheader-allowed-names"] = deps.RequestHeaderCN argsMap["proxy-client-cert-file"] = runtime.ClientAuthProxyCert @@ -199,7 +217,15 @@ func apiServer(ctx context.Context, cfg *config.Control) error { argsMap["profiling"] = "false" if cfg.EncryptSecrets { argsMap["encryption-provider-config"] = runtime.EncryptionConfig + argsMap["encryption-provider-config-automatic-reload"] = "true" + } + if cfg.VLevel != 0 { + argsMap["v"] = strconv.Itoa(cfg.VLevel) + } + if cfg.VModule != "" { + argsMap["vmodule"] = cfg.VModule } + args := config.GetArgs(argsMap, cfg.ExtraAPIArgs) logrus.Infof("Running kube-apiserver %s", config.ArgString(args)) @@ -246,7 +272,6 @@ func prepare(ctx context.Context, config *config.Control) error { deps.CreateRuntimeCertFiles(config) cluster := cluster.New(config) - if err := cluster.Bootstrap(ctx, config.ClusterReset); err != nil { return err } @@ -298,6 +323,7 @@ func cloudControllerManager(ctx context.Context, cfg *config.Control) error { "authentication-kubeconfig": runtime.KubeConfigCloudController, "node-status-update-frequency": "1m0s", "bind-address": cfg.Loopback(false), + "feature-gates": "CloudDualStackNodeIPs=true", } if cfg.NoLeaderElect { argsMap["leader-elect"] = "false" @@ -306,8 +332,16 @@ func cloudControllerManager(ctx context.Context, cfg *config.Control) error { argsMap["controllers"] = argsMap["controllers"] + ",-cloud-node,-cloud-node-lifecycle" argsMap["secure-port"] = "0" } + if cfg.DisableServiceLB { + argsMap["controllers"] = argsMap["controllers"] + ",-service" + } + if cfg.VLevel != 0 { + argsMap["v"] = strconv.Itoa(cfg.VLevel) + } + if cfg.VModule != "" { + argsMap["vmodule"] = cfg.VModule + } - argsMap["controllers"] = argsMap["controllers"] + ",-service" args := config.GetArgs(argsMap, cfg.ExtraCloudControllerArgs) logrus.Infof("Running cloud-controller-manager %s", config.ArgString(args)) @@ -414,4 +448,4 @@ func promise(f func() error) <-chan error { close(c) }() return c -} +} \ No newline at end of file diff --git a/pkg/daemons/control/tunnel.go b/pkg/daemons/control/tunnel.go index 6de49012..cb0ab31c 100644 --- a/pkg/daemons/control/tunnel.go +++ b/pkg/daemons/control/tunnel.go @@ -10,21 +10,16 @@ import ( "strings" "sync" - "github.com/pkg/errors" - "github.com/rancher/remotedialer" - "github.com/sirupsen/logrus" "github.com/xiaods/k8e/pkg/daemons/config" "github.com/xiaods/k8e/pkg/daemons/control/proxy" - "github.com/xiaods/k8e/pkg/generated/clientset/versioned/scheme" "github.com/xiaods/k8e/pkg/nodeconfig" "github.com/xiaods/k8e/pkg/util" "github.com/xiaods/k8e/pkg/version" + "github.com/pkg/errors" + "github.com/rancher/remotedialer" + "github.com/sirupsen/logrus" "github.com/yl2chen/cidranger" v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/client-go/kubernetes" ) @@ -33,8 +28,7 @@ var defaultDialer = net.Dialer{} func loggingErrorWriter(rw http.ResponseWriter, req *http.Request, code int, err error) { logrus.Debugf("Tunnel server error: %d %v", code, err) - rw.WriteHeader(code) - rw.Write([]byte(err.Error())) + util.SendError(err, rw, req, code) } func setupTunnel(ctx context.Context, cfg *config.Control) (http.Handler, error) { @@ -172,29 +166,20 @@ func (t *TunnelServer) onChangePod(podName string, pod *v1.Pod) (*v1.Pod, error) func (t *TunnelServer) serveConnect(resp http.ResponseWriter, req *http.Request) { bconn, err := t.dialBackend(req.Context(), req.Host) if err != nil { - responsewriters.ErrorNegotiated( - newBadGateway(err.Error()), - scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req, - ) + util.SendError(err, resp, req, http.StatusBadGateway) return } hijacker, ok := resp.(http.Hijacker) if !ok { - responsewriters.ErrorNegotiated( - apierrors.NewInternalError(errors.New("hijacking not supported")), - scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req, - ) + util.SendError(errors.New("hijacking not supported"), resp, req, http.StatusInternalServerError) return } resp.WriteHeader(http.StatusOK) rconn, bufrw, err := hijacker.Hijack() if err != nil { - responsewriters.ErrorNegotiated( - apierrors.NewInternalError(err), - scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req, - ) + util.SendError(err, resp, req, http.StatusInternalServerError) return } @@ -211,7 +196,6 @@ func (t *TunnelServer) dialBackend(ctx context.Context, addr string) (net.Conn, if err != nil { return nil, err } - loopback := t.config.Loopback(true) var nodeName string var toKubelet, useTunnel bool @@ -238,14 +222,17 @@ func (t *TunnelServer) dialBackend(ctx context.Context, addr string) (net.Conn, useTunnel = true } - // Always dial kubelet via the loopback address. - if toKubelet { - addr = net.JoinHostPort(loopback, port) - } - // If connecting to something hosted by the local node, don't tunnel if nodeName == t.config.ServerNodeName { useTunnel = false + if toKubelet { + // Dial local kubelet at the configured bind address + addr = net.JoinHostPort(t.config.BindAddress, port) + } + } else if toKubelet { + // Dial remote kubelet via the loopback address, the remotedialer client + // will ensure that it hits the right local address. + addr = net.JoinHostPort(t.config.Loopback(false), port) } if useTunnel { @@ -299,15 +286,4 @@ func (crw *connReadWriteCloser) Write(b []byte) (n int, err error) { func (crw *connReadWriteCloser) Close() (err error) { crw.once.Do(func() { err = crw.conn.Close() }) return -} - -func newBadGateway(message string) *apierrors.StatusError { - return &apierrors.StatusError{ - ErrStatus: metav1.Status{ - Status: metav1.StatusFailure, - Code: http.StatusBadGateway, - Reason: metav1.StatusReasonInternalError, - Message: message, - }, - } -} +} \ No newline at end of file diff --git a/pkg/daemons/executor/embed_linux.go b/pkg/daemons/executor/embed_linux.go deleted file mode 100644 index a1224581..00000000 --- a/pkg/daemons/executor/embed_linux.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build linux && !no_embedded_executor -// +build linux,!no_embedded_executor - -package executor - -import ( - // registering k8e cloud provider - _ "github.com/xiaods/k8e/pkg/cloudprovider" -) - diff --git a/pkg/daemons/executor/embed_windows.go b/pkg/daemons/executor/embed_windows.go deleted file mode 100644 index 4e935078..00000000 --- a/pkg/daemons/executor/embed_windows.go +++ /dev/null @@ -1,81 +0,0 @@ -//go:build windows && !no_embedded_executor -// +build windows,!no_embedded_executor - -package executor - -import ( - "encoding/json" - "os" - "os/exec" - "strings" - "time" - - "github.com/Microsoft/hcsshim" - "github.com/sirupsen/logrus" - - // registering k8e cloud provider - _ "github.com/xiaods/k8e/pkg/cloudprovider" - daemonconfig "github.com/xiaods/k8e/pkg/daemons/config" -) - -const ( - networkName = "flannel.4096" -) - -type SourceVipResponse struct { - IP4 struct { - IP string `json:"ip"` - } `json:"ip4"` -} - -func waitForSourceVip(networkName string, nodeConfig *daemonconfig.Node) string { - for range time.Tick(time.Second * 5) { - network, err := hcsshim.GetHNSNetworkByName(networkName) - if err != nil { - logrus.WithError(err).Warningf("can't find HNS network, retrying %s", networkName) - continue - } - if network.ManagementIP == "" { - logrus.WithError(err).Warningf("wait for management IP, retrying %s", networkName) - continue - } - - subnet := network.Subnets[0].AddressPrefix - - configData := `{ - "cniVersion": "0.2.0", - "name": "vxlan0", - "ipam": { - "type": "host-local", - "ranges": [[{"subnet":"` + subnet + `"}]], - "dataDir": "/var/lib/cni/networks" - } - }` - - cmd := exec.Command("host-local.exe") - cmd.Env = append(os.Environ(), - "CNI_COMMAND=ADD", - "CNI_CONTAINERID=dummy", - "CNI_NETNS=dummy", - "CNI_IFNAME=dummy", - "CNI_PATH="+nodeConfig.AgentConfig.CNIBinDir, - ) - - cmd.Stdin = strings.NewReader(configData) - out, err := cmd.Output() - if err != nil { - logrus.WithError(err).Warning("Failed to execute host-local.exe") - continue - } - - var sourceVipResp SourceVipResponse - err = json.Unmarshal(out, &sourceVipResp) - if err != nil { - logrus.WithError(err).Warning("Failed to unmarshal sourceVip response") - continue - } - - return strings.TrimSpace(strings.Split(sourceVipResp.IP4.IP, "/")[0]) - } - return "" -} \ No newline at end of file