Dev

go.mod

@@ -74,7 +74,6 @@ replace (
 )
 
 require (
-	github.com/Microsoft/hcsshim v0.12.6
 	github.com/Mirantis/cri-dockerd v0.0.0-00010101000000-000000000000
 	github.com/blang/semver/v4 v4.0.0
 	github.com/containerd/aufs v1.0.0
@@ -183,6 +182,7 @@ require (
 	github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/Microsoft/hcsshim v0.12.6 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/Rican7/retry v0.1.0 // indirect
 	github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e // indirect

pkg/agent/templates/templates_linux.go

@@ -62,12 +62,6 @@ enable_keychain = true
 {{end}}
 {{end}}
 
-{{- if not .NodeConfig.NoFlannel }}
-[plugins."io.containerd.grpc.v1.cri".cni]
-  bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
-  conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
-{{end}}
-
 {{- if or .NodeConfig.Containerd.BlockIOConfig .NodeConfig.Containerd.RDTConfig }}
 [plugins."io.containerd.service.v1.tasks-service"]
   {{ if .NodeConfig.Containerd.BlockIOConfig }}blockio_config_file = "{{ .NodeConfig.Containerd.BlockIOConfig }}"{{end}}

pkg/cli/cmds/agent.go

@@ -15,7 +15,6 @@ type Agent struct {
 	ServerURL                string
 	APIAddressCh             chan []string
 	DisableLoadBalancer      bool
-	DisableServiceLB         bool
 	ETCDAgent                bool
 	LBServerPort             int
 	ResolvConf               string

pkg/cli/cmds/server.go

@@ -104,7 +104,6 @@ type Server struct {
 	EtcdS3ConfigSecret       string
 	EtcdS3Timeout            time.Duration
 	EtcdS3Insecure           bool
-	ServiceLBNamespace       string
 }
 
 var (
@@ -216,12 +215,6 @@ var ServerFlags = []cli.Flag{
 		Destination: &ServerConfig.EgressSelectorMode,
 		Value:       "agent",
 	},
-	&cli.StringFlag{
-		Name:        "servicelb-namespace",
-		Usage:       "(networking) Namespace of the pods for the servicelb component",
-		Destination: &ServerConfig.ServiceLBNamespace,
-		Value:       "kube-system",
-	},
 	&cli.StringFlag{
 		Name:        "write-kubeconfig,o",
 		Usage:       "(client) Write kubeconfig for admin client to this file",
@@ -510,13 +503,9 @@ var ServerFlags = []cli.Flag{
 	NodeIPFlag,
 	NodeExternalIPFlag,
 	ResolvConfFlag,
-	FlannelIfaceFlag,
-	FlannelConfFlag,
-	FlannelCniConfFileFlag,
 	VPNAuth,
 	VPNAuthFile,
 	ExtraKubeletArgs,
-	ExtraKubeProxyArgs,
 	ProtectKernelDefaultsFlag,
 	&cli.BoolFlag{
 		Name:        "secrets-encryption",

pkg/daemons/config/types.go

@@ -188,7 +188,6 @@ type Control struct {
 	DisableETCD              bool
 	DisableScheduler         bool
 	Rootless                 bool
-	ServiceLBNamespace       string
 	ExtraAPIArgs             []string
 	ExtraControllerArgs      []string
 	ExtraCloudControllerArgs []string

pkg/daemons/control/server.go

@@ -20,8 +20,9 @@ import (
 	"github.com/xiaods/k8e/pkg/version"
 	authorizationv1 "k8s.io/api/authorization/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	logsapi "k8s.io/component-base/logs/api/v1"
 	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
-	proxyutil "k8s.io/kubernetes/pkg/proxy/util"
+	"k8s.io/kubernetes/pkg/registry/core/node"
 
 	// for client metric registration
 	_ "k8s.io/component-base/metrics/prometheus/restclient"
@@ -30,6 +31,7 @@ import (
 func Server(ctx context.Context, cfg *config.Control) error {
 	rand.Seed(time.Now().UTC().UnixNano())
 
+	logsapi.ReapplyHandling = logsapi.ReapplyHandlingIgnoreUnchanged
 	if err := prepare(ctx, cfg); err != nil {
 		return errors.Wrap(err, "preparing server")
 	}
@@ -40,7 +42,7 @@ func Server(ctx context.Context, cfg *config.Control) error {
 	}
 	cfg.Runtime.Tunnel = tunnel
 
-	proxyutil.DisableProxyHostnameCheck = true
+	node.DisableProxyHostnameCheck = true
 
 	authArgs := []string{
 		"--basic-auth-file=" + cfg.Runtime.PasswdFile,
@@ -90,7 +92,6 @@ func controllerManager(ctx context.Context, cfg *config.Control) error {
 	runtime := cfg.Runtime
 	argsMap := map[string]string{
 		"controllers":                      "*,tokencleaner",
-		"feature-gates":                    "JobTrackingWithFinalizers=true",
 		"kubeconfig":                       runtime.KubeConfigController,
 		"authorization-kubeconfig":         runtime.KubeConfigController,
 		"authentication-kubeconfig":        runtime.KubeConfigController,
@@ -120,6 +121,13 @@ func controllerManager(ctx context.Context, cfg *config.Control) error {
 		argsMap["controllers"] = argsMap["controllers"] + ",-service,-route,-cloud-node-lifecycle"
 	}
 
+	if cfg.VLevel != 0 {
+		argsMap["v"] = strconv.Itoa(cfg.VLevel)
+	}
+	if cfg.VModule != "" {
+		argsMap["vmodule"] = cfg.VModule
+	}
+
 	args := config.GetArgs(argsMap, cfg.ExtraControllerArgs)
 	logrus.Infof("Running kube-controller-manager %s", config.ArgString(args))
 
@@ -139,6 +147,14 @@ func scheduler(ctx context.Context, cfg *config.Control) error {
 	if cfg.NoLeaderElect {
 		argsMap["leader-elect"] = "false"
 	}
+
+	if cfg.VLevel != 0 {
+		argsMap["v"] = strconv.Itoa(cfg.VLevel)
+	}
+	if cfg.VModule != "" {
+		argsMap["vmodule"] = cfg.VModule
+	}
+
 	args := config.GetArgs(argsMap, cfg.ExtraSchedulerAPIArgs)
 
 	logrus.Infof("Running kube-scheduler %s", config.ArgString(args))
@@ -147,9 +163,7 @@ func scheduler(ctx context.Context, cfg *config.Control) error {
 
 func apiServer(ctx context.Context, cfg *config.Control) error {
 	runtime := cfg.Runtime
-	argsMap := map[string]string{
-		"feature-gates": "JobTrackingWithFinalizers=true",
-	}
+	argsMap := map[string]string{}
 
 	setupStorageBackend(argsMap, cfg)
 
@@ -186,6 +200,7 @@ func apiServer(ctx context.Context, cfg *config.Control) error {
 	argsMap["kubelet-client-certificate"] = runtime.ClientKubeAPICert
 	argsMap["kubelet-client-key"] = runtime.ClientKubeAPIKey
 	argsMap["kubelet-preferred-address-types"] = "InternalIP,ExternalIP,Hostname"
+
 	argsMap["requestheader-client-ca-file"] = runtime.RequestHeaderCA
 	argsMap["requestheader-allowed-names"] = deps.RequestHeaderCN
 	argsMap["proxy-client-cert-file"] = runtime.ClientAuthProxyCert
@@ -199,7 +214,15 @@ func apiServer(ctx context.Context, cfg *config.Control) error {
 	argsMap["profiling"] = "false"
 	if cfg.EncryptSecrets {
 		argsMap["encryption-provider-config"] = runtime.EncryptionConfig
+		argsMap["encryption-provider-config-automatic-reload"] = "true"
+	}
+	if cfg.VLevel != 0 {
+		argsMap["v"] = strconv.Itoa(cfg.VLevel)
 	}
+	if cfg.VModule != "" {
+		argsMap["vmodule"] = cfg.VModule
+	}
+
 	args := config.GetArgs(argsMap, cfg.ExtraAPIArgs)
 
 	logrus.Infof("Running kube-apiserver %s", config.ArgString(args))
@@ -246,7 +269,6 @@ func prepare(ctx context.Context, config *config.Control) error {
 	deps.CreateRuntimeCertFiles(config)
 
 	cluster := cluster.New(config)
-
 	if err := cluster.Bootstrap(ctx, config.ClusterReset); err != nil {
 		return err
 	}
@@ -298,6 +320,7 @@ func cloudControllerManager(ctx context.Context, cfg *config.Control) error {
 		"authentication-kubeconfig":    runtime.KubeConfigCloudController,
 		"node-status-update-frequency": "1m0s",
 		"bind-address":                 cfg.Loopback(false),
+		"feature-gates":                "CloudDualStackNodeIPs=true",
 	}
 	if cfg.NoLeaderElect {
 		argsMap["leader-elect"] = "false"
@@ -306,8 +329,13 @@ func cloudControllerManager(ctx context.Context, cfg *config.Control) error {
 		argsMap["controllers"] = argsMap["controllers"] + ",-cloud-node,-cloud-node-lifecycle"
 		argsMap["secure-port"] = "0"
 	}
+	if cfg.VLevel != 0 {
+		argsMap["v"] = strconv.Itoa(cfg.VLevel)
+	}
+	if cfg.VModule != "" {
+		argsMap["vmodule"] = cfg.VModule
+	}
 
-	argsMap["controllers"] = argsMap["controllers"] + ",-service"
 	args := config.GetArgs(argsMap, cfg.ExtraCloudControllerArgs)
 
 	logrus.Infof("Running cloud-controller-manager %s", config.ArgString(args))
@@ -414,4 +442,4 @@ func promise(f func() error) <-chan error {
 		close(c)
 	}()
 	return c
-}
+}

pkg/daemons/control/tunnel.go

@@ -10,21 +10,16 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/pkg/errors"
-	"github.com/rancher/remotedialer"
-	"github.com/sirupsen/logrus"
 	"github.com/xiaods/k8e/pkg/daemons/config"
 	"github.com/xiaods/k8e/pkg/daemons/control/proxy"
-	"github.com/xiaods/k8e/pkg/generated/clientset/versioned/scheme"
 	"github.com/xiaods/k8e/pkg/nodeconfig"
 	"github.com/xiaods/k8e/pkg/util"
 	"github.com/xiaods/k8e/pkg/version"
+	"github.com/pkg/errors"
+	"github.com/rancher/remotedialer"
+	"github.com/sirupsen/logrus"
 	"github.com/yl2chen/cidranger"
 	v1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/client-go/kubernetes"
 )
@@ -33,8 +28,7 @@ var defaultDialer = net.Dialer{}
 
 func loggingErrorWriter(rw http.ResponseWriter, req *http.Request, code int, err error) {
 	logrus.Debugf("Tunnel server error: %d %v", code, err)
-	rw.WriteHeader(code)
-	rw.Write([]byte(err.Error()))
+	util.SendError(err, rw, req, code)
 }
 
 func setupTunnel(ctx context.Context, cfg *config.Control) (http.Handler, error) {
@@ -172,29 +166,20 @@ func (t *TunnelServer) onChangePod(podName string, pod *v1.Pod) (*v1.Pod, error)
 func (t *TunnelServer) serveConnect(resp http.ResponseWriter, req *http.Request) {
 	bconn, err := t.dialBackend(req.Context(), req.Host)
 	if err != nil {
-		responsewriters.ErrorNegotiated(
-			newBadGateway(err.Error()),
-			scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
-		)
+		util.SendError(err, resp, req, http.StatusBadGateway)
 		return
 	}
 
 	hijacker, ok := resp.(http.Hijacker)
 	if !ok {
-		responsewriters.ErrorNegotiated(
-			apierrors.NewInternalError(errors.New("hijacking not supported")),
-			scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
-		)
+		util.SendError(errors.New("hijacking not supported"), resp, req, http.StatusInternalServerError)
 		return
 	}
 	resp.WriteHeader(http.StatusOK)
 
 	rconn, bufrw, err := hijacker.Hijack()
 	if err != nil {
-		responsewriters.ErrorNegotiated(
-			apierrors.NewInternalError(err),
-			scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req,
-		)
+		util.SendError(err, resp, req, http.StatusInternalServerError)
 		return
 	}
 
@@ -211,7 +196,6 @@ func (t *TunnelServer) dialBackend(ctx context.Context, addr string) (net.Conn,
 	if err != nil {
 		return nil, err
 	}
-	loopback := t.config.Loopback(true)
 
 	var nodeName string
 	var toKubelet, useTunnel bool
@@ -238,14 +222,17 @@ func (t *TunnelServer) dialBackend(ctx context.Context, addr string) (net.Conn,
 		useTunnel = true
 	}
 
-	// Always dial kubelet via the loopback address.
-	if toKubelet {
-		addr = net.JoinHostPort(loopback, port)
-	}
-
 	// If connecting to something hosted by the local node, don't tunnel
 	if nodeName == t.config.ServerNodeName {
 		useTunnel = false
+		if toKubelet {
+			// Dial local kubelet at the configured bind address
+			addr = net.JoinHostPort(t.config.BindAddress, port)
+		}
+	} else if toKubelet {
+		// Dial remote kubelet via the loopback address, the remotedialer client
+		// will ensure that it hits the right local address.
+		addr = net.JoinHostPort(t.config.Loopback(false), port)
 	}
 
 	if useTunnel {
@@ -299,15 +286,4 @@ func (crw *connReadWriteCloser) Write(b []byte) (n int, err error) {
 func (crw *connReadWriteCloser) Close() (err error) {
 	crw.once.Do(func() { err = crw.conn.Close() })
 	return
-}
-
-func newBadGateway(message string) *apierrors.StatusError {
-	return &apierrors.StatusError{
-		ErrStatus: metav1.Status{
-			Status:  metav1.StatusFailure,
-			Code:    http.StatusBadGateway,
-			Reason:  metav1.StatusReasonInternalError,
-			Message: message,
-		},
-	}
-}
+}