fix: prevent the goroutine leak from lb health checks

We had a leak of goroutines where the `lb.Start()` failed due to a port conflict. The health check goroutines were never terminated, were waiting on the context. This caused a leak of goroutines over a long time (as every failure on `.Start()` caused an exponential backoff) and increased CPU usage. Fix this by calling `lb.Close()` even if starting the load balancer has failed. Additionally, add the following features: - allow running an optional pprof server - allow specifying disallowed host port ranges to not be attempted to be proxied Also rekres & bump deps. Signed-off-by: Utku Ozdemir <[email protected]>
siderolabs · Aug 14, 2024 · a09759e · a09759e
1 parent f582988
commit a09759e
Show file tree

Hide file tree

Showing 19 changed files with 348 additions and 139 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -2,7 +2,7 @@
 
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-07-25T21:33:47Z by kres faf91e3.
+# Generated on 2024-08-14T09:22:15Z by kres 7be2a05.
 
 ARG TOOLCHAIN
 
@@ -11,7 +11,7 @@ FROM ghcr.io/siderolabs/ca-certificates:v1.7.0 AS image-ca-certificates
 FROM ghcr.io/siderolabs/fhs:v1.7.0 AS image-fhs
 
 # runs markdownlint
-FROM docker.io/oven/bun:1.1.20-alpine AS lint-markdown
+FROM docker.io/oven/bun:1.1.22-alpine AS lint-markdown
 WORKDIR /src
 RUN bun i [email protected] [email protected]
 COPY .markdownlint.json .

diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-07-25T21:33:47Z by kres faf91e3.
+# Generated on 2024-08-14T09:22:15Z by kres 7be2a05.
 
 # common variables
 
@@ -18,14 +18,14 @@ REGISTRY ?= ghcr.io
 USERNAME ?= siderolabs
 REGISTRY_AND_USERNAME ?= $(REGISTRY)/$(USERNAME)
 PROTOBUF_GO_VERSION ?= 1.34.2
-GRPC_GO_VERSION ?= 1.4.0
-GRPC_GATEWAY_VERSION ?= 2.20.0
+GRPC_GO_VERSION ?= 1.5.1
+GRPC_GATEWAY_VERSION ?= 2.21.0
 VTPROTOBUF_VERSION ?= 0.6.0
-GOIMPORTS_VERSION ?= 0.23.0
+GOIMPORTS_VERSION ?= 0.24.0
 DEEPCOPY_VERSION ?= v0.5.6
-GOLANGCILINT_VERSION ?= v1.59.1
+GOLANGCILINT_VERSION ?= v1.60.1
 GOFUMPT_VERSION ?= v0.6.0
-GO_VERSION ?= 1.22.5
+GO_VERSION ?= 1.23.0
 GO_BUILDFLAGS ?=
 GO_LDFLAGS ?=
 CGO_ENABLED ?= 0
@@ -67,7 +67,7 @@ COMMON_ARGS += --build-arg=DEEPCOPY_VERSION="$(DEEPCOPY_VERSION)"
 COMMON_ARGS += --build-arg=GOLANGCILINT_VERSION="$(GOLANGCILINT_VERSION)"
 COMMON_ARGS += --build-arg=GOFUMPT_VERSION="$(GOFUMPT_VERSION)"
 COMMON_ARGS += --build-arg=TESTPKGS="$(TESTPKGS)"
-TOOLCHAIN ?= docker.io/golang:1.22-alpine
+TOOLCHAIN ?= docker.io/golang:1.23-alpine
 
 # help menu
 

diff --git a/cmd/kube-service-exposer/main.go b/cmd/kube-service-exposer/main.go
@@ -6,13 +6,19 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"log"
+	"net/http"
+	"net/http/pprof"
+	"time"
 
 	"github.com/go-logr/zapr"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
+	"golang.org/x/sync/errgroup"
 	controllerruntimelog "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 
@@ -22,8 +28,10 @@ import (
 )
 
 var rootCmdArgs struct {
-	annotationKey string
-	bindCIDRs     []string
+	annotationKey            string
+	pprofBindAddr            string
+	bindCIDRs                []string
+	disallowedHostPortRanges []string
 
 	debug bool
 }
@@ -57,12 +65,24 @@ var rootCmd = &cobra.Command{
 
 		controllerruntimelog.SetLogger(zapr.NewLogger(logger))
 
-		exposer, err := exposer.New(rootCmdArgs.annotationKey, rootCmdArgs.bindCIDRs, logger.With(zap.String("component", "exposer")))
+		exposer, err := exposer.New(rootCmdArgs.annotationKey, rootCmdArgs.bindCIDRs, rootCmdArgs.disallowedHostPortRanges, logger.With(zap.String("component", "exposer")))
 		if err != nil {
 			return err
 		}
 
-		return exposer.Run(cmd.Context())
+		eg, ctx := errgroup.WithContext(cmd.Context())
+
+		eg.Go(func() error {
+			return exposer.Run(ctx)
+		})
+
+		if rootCmdArgs.pprofBindAddr != "" {
+			eg.Go(func() error {
+				return runPprofServer(ctx, logger)
+			})
+		}
+
+		return eg.Wait()
 	},
 }
 
@@ -74,10 +94,56 @@ func main() {
 	}
 }
 
+func runPprofServer(ctx context.Context, logger *zap.Logger) error {
+	logger.Info("starting pprof server", zap.String("addr", rootCmdArgs.pprofBindAddr))
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/debug/pprof/", pprof.Index)
+	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+	mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+	mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+
+	server := &http.Server{
+		Addr:    rootCmdArgs.pprofBindAddr,
+		Handler: mux,
+	}
+
+	errCh := make(chan error, 1)
+
+	go func() { errCh <- server.ListenAndServe() }()
+
+	select {
+	case err := <-errCh:
+		if err != nil && !errors.Is(err, http.ErrServerClosed) {
+			return fmt.Errorf("failed to serve: %w", err)
+		}
+
+		return nil
+	case <-ctx.Done():
+	}
+
+	logger.Info("stopping pprof server")
+
+	shutdownCtx, shutdownCtxCancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer shutdownCtxCancel()
+
+	//nolint:contextcheck
+	if err := server.Shutdown(shutdownCtx); err != nil {
+		return fmt.Errorf("failed to shutdown pprof server gracefully: %w", err)
+	}
+
+	return nil
+}
+
 func init() {
 	rootCmd.Flags().StringVarP(&rootCmdArgs.annotationKey, "annotation-key", "a", version.Name+".sidero.dev/port",
 		"the annotation key to be looked for on the services to determine which port to expose ot from.")
-	rootCmd.Flags().StringSliceVarP(&rootCmdArgs.bindCIDRs, "bind-cidrs", "b", []string{},
+	rootCmd.Flags().StringVar(&rootCmdArgs.pprofBindAddr, "pprof-bind-addr", "",
+		"the address to bind the pprof server to. Disabled when empty.")
+	rootCmd.Flags().StringSliceVarP(&rootCmdArgs.bindCIDRs, "bind-cidrs", "b", nil,
 		"the CIDRs to match the host IPs with. Only the ports on the IPs that match these CIDRs will be listened. When empty, all IPs will be listened.")
+	rootCmd.Flags().StringSliceVar(&rootCmdArgs.disallowedHostPortRanges, "disallowed-host-port-ranges", nil,
+		"the port ranges on the host that are not allowed to be used. When a disallowed host port is attempted to be exposed, it will be skipped and a warning will be logged.")
 	rootCmd.Flags().BoolVar(&rootCmdArgs.debug, "debug", false, "enable debug logs.")
 }
diff --git a/deploy/kube-service-exposer.yaml b/deploy/kube-service-exposer.yaml
@@ -47,8 +47,9 @@ spec:
         - operator: Exists
       containers:
         - name: kube-service-exposer
-          image: ghcr.io/siderolabs/kube-service-exposer:v0.1.0
+          image: ghcr.io/siderolabs/kube-service-exposer:v0.2.0
           # args:
           #   - --debug=true
+          #   - --pprof-bind-addr=:6060
           #   - --annotation-key=my-annotation-key/port
           #   - --bind-cidrs=172.20.0.0/24
diff --git a/go.mod b/go.mod
@@ -1,6 +1,6 @@
 module github.com/siderolabs/kube-service-exposer
 
-go 1.22.3
+go 1.23.0
 
 require (
 	github.com/benbjohnson/clock v1.3.5
@@ -12,17 +12,18 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0
 	go.uber.org/zap v1.27.0
-	golang.org/x/sync v0.7.0
-	k8s.io/api v0.30.2
-	k8s.io/apimachinery v0.30.2
-	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
-	sigs.k8s.io/controller-runtime v0.18.4
+	golang.org/x/sync v0.8.0
+	k8s.io/api v0.30.3
+	k8s.io/apimachinery v0.30.3
+	k8s.io/client-go v0.30.3
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
+	sigs.k8s.io/controller-runtime v0.18.5
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.12.1 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
@@ -48,29 +49,28 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.54.0 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/siderolabs/tcpproxy v0.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/term v0.21.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
+	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/oauth2 v0.22.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/term v0.23.0 // indirect
+	golang.org/x/text v0.17.0 // indirect
+	golang.org/x/time v0.6.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/client-go v0.30.2 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect
+	k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect