Implement shoot migration.

api/v1/helper/scheme.go

@@ -0,0 +1,29 @@
+package helper
+
+import (
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+
+	firewallv2 "github.com/metal-stack/firewall-controller-manager/api/v2"
+	firewallv1 "github.com/metal-stack/firewall-controller/api/v1"
+)
+
+var (
+	scheme = runtime.NewScheme()
+)
+
+func init() {
+	_ = clientgoscheme.AddToScheme(scheme)
+
+	_ = firewallv1.AddToScheme(scheme)
+	_ = firewallv2.AddToScheme(scheme)
+
+	_ = apiextensions.AddToScheme(scheme)
+	// +kubebuilder:scaffold:scheme
+}
+
+func Scheme() *runtime.Scheme {
+	return scheme
+}

controllers/firewall_monitor_controller.go

@@ -3,16 +3,22 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/go-logr/logr"
 	firewallv2 "github.com/metal-stack/firewall-controller-manager/api/v2"
 	firewallv1 "github.com/metal-stack/firewall-controller/api/v1"
+	apihelper "github.com/metal-stack/firewall-controller/api/v1/helper"
 	"github.com/metal-stack/firewall-controller/pkg/collector"
 	"github.com/metal-stack/firewall-controller/pkg/suricata"
 	"github.com/metal-stack/v"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/clientcmd"
+	configlatest "k8s.io/client-go/tools/clientcmd/api/latest"
+	configv1 "k8s.io/client-go/tools/clientcmd/api/v1"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -30,6 +36,9 @@ type FirewallMonitorReconciler struct {
 	FirewallName string
 	Namespace    string
 
+	SeedNamespace      string
+	SeedKubeconfigPath string
+
 	IDSEnabled bool
 	Interval   time.Duration
 }
@@ -75,6 +84,10 @@ func (r *FirewallMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		return ctrl.Result{}, nil
 	}
 
+	if err := r.checkSeedEndpoint(ctx, mon); err != nil {
+		return ctrl.Result{}, err
+	}
+
 	c := collector.NewNFTablesCollector(&r.Log)
 	ruleStats := c.CollectRuleStats()
 
@@ -123,3 +136,98 @@ func (r *FirewallMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		RequeueAfter: r.Interval,
 	}, nil
 }
+
+func (r *FirewallMonitorReconciler) checkSeedEndpoint(ctx context.Context, mon *firewallv2.FirewallMonitor) error {
+	seedURL, ok := mon.Annotations[firewallv2.FirewallSeedURLAnnotation]
+	if !ok {
+		return nil
+	}
+
+	rawKubeconfig, err := os.ReadFile(r.SeedKubeconfigPath)
+	if err != nil {
+		return fmt.Errorf("unable to read seed kubeconfig: %w", err)
+	}
+
+	seedConfig, err := clientcmd.RESTConfigFromKubeConfig(rawKubeconfig)
+	if err != nil {
+		return fmt.Errorf("unable to create rest config from seed kubeconfig: %w", err)
+	}
+
+	if seedConfig.APIPath == seedURL {
+		return nil
+	}
+
+	r.Log.Info("seed api url is different in firewall monitor annotation, testing current seed client", "current-url", seedConfig.APIPath, "annotation-url", seedURL)
+
+	clientTest := func(c client.Client) error {
+		f := &firewallv2.Firewall{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      mon.Name,
+				Namespace: r.SeedNamespace,
+			},
+		}
+
+		return c.Get(ctx, client.ObjectKeyFromObject(f), f)
+	}
+
+	seedClient, err := client.New(seedConfig, client.Options{
+		Scheme: apihelper.Scheme(),
+	})
+	if err != nil {
+		return fmt.Errorf("unable to create seed client from seed kubeconfig: %w", err)
+	}
+
+	err = clientTest(seedClient)
+	if err == nil {
+		r.Log.Info("current seed client seems to work, not taking any further actions")
+		return nil
+	}
+
+	r.Log.Error(err, "current seed client seems not to work, attemping seed client update")
+
+	kubeconfig := &configv1.Config{}
+	err = runtime.DecodeInto(configlatest.Codec, rawKubeconfig, kubeconfig)
+	if err != nil {
+		return fmt.Errorf("unable to decode kubeconfig seed kubeconfig: %w", err)
+	}
+
+	for _, cluster := range kubeconfig.Clusters {
+		cluster := cluster
+		cluster.Cluster.Server = seedURL
+	}
+
+	updatedKubeconfig, err := runtime.Encode(configlatest.Codec, kubeconfig)
+	if err != nil {
+		return fmt.Errorf("unable to encode kubeconfig: %w", err)
+	}
+
+	updatedConfig, err := clientcmd.RESTConfigFromKubeConfig(updatedKubeconfig)
+	if err != nil {
+		return fmt.Errorf("unable to create rest config from bytes: %w", err)
+	}
+
+	newSeedClient, err := client.New(updatedConfig, client.Options{
+		Scheme: apihelper.Scheme(),
+	})
+	if err != nil {
+		return fmt.Errorf("unable to create seed client from updated seed kubeconfig: %w", err)
+	}
+
+	err = clientTest(newSeedClient)
+	if err != nil {
+		return fmt.Errorf("seed client seems broken but seed client with changed api server url also does not appear to work, seed connection lost?")
+	}
+
+	err = os.WriteFile(r.SeedKubeconfigPath, updatedKubeconfig, 0600)
+	if err != nil {
+		return fmt.Errorf("unable to write kubeconfig to destination: %w", err)
+	}
+
+	r.Log.Info("successfully updating seed client url, restarting controller")
+
+	// not sure if there is a more graceful way to shutdown this controller?
+	os.Exit(0)
+
+	return nil
+
+}

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/google/nftables v0.1.0
 	github.com/ks2211/go-suricata v0.0.0-20200823200910-986ce1470707
-	github.com/metal-stack/firewall-controller-manager v0.2.2
+	github.com/metal-stack/firewall-controller-manager v0.2.3-0.20230620144431-5175cc6726bb
 	github.com/metal-stack/gardener-extension-provider-metal v0.20.3
 	github.com/metal-stack/metal-go v0.22.6
 	github.com/metal-stack/metal-lib v0.11.10

go.sum

@@ -229,8 +229,8 @@ github.com/mdlayher/netlink v1.7.0 h1:ZNGI4V7i1fJ94DPYtWhI/R85i/Q7ZxnuhUJQcJMood
 github.com/mdlayher/netlink v1.7.0/go.mod h1:nKO5CSjE/DJjVhk/TNp6vCE1ktVxEA8VEh8drhZzxsQ=
 github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw=
 github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc=
-github.com/metal-stack/firewall-controller-manager v0.2.2 h1:PM60rnyIb38EVnm4MEm8y2AGeiqwopf8h707qaTJsso=
-github.com/metal-stack/firewall-controller-manager v0.2.2/go.mod h1:f4c+yYBQLuNpS5cyGXBj7c0b5TeR770LaElgPgA6DTw=
+github.com/metal-stack/firewall-controller-manager v0.2.3-0.20230620144431-5175cc6726bb h1:vYRHLxaNMIc1S376MA9a0i0EFaDbAjTYIGSjrrf43V8=
+github.com/metal-stack/firewall-controller-manager v0.2.3-0.20230620144431-5175cc6726bb/go.mod h1:f4c+yYBQLuNpS5cyGXBj7c0b5TeR770LaElgPgA6DTw=
 github.com/metal-stack/gardener-extension-provider-metal v0.20.3 h1:hhNLjACU2vYbZJFx7XuFXEAZXgXKElq6Bb5FFFUJEiQ=
 github.com/metal-stack/gardener-extension-provider-metal v0.20.3/go.mod h1:r0SgbEF3au3pJCMmriA3PNaawUd9h3v8msrMt43rGxI=
 github.com/metal-stack/metal-go v0.22.6 h1:nWjk2VH1B1CWGoupYGxxceblWszTjpD08729riaRyUE=

main.go

@@ -15,12 +15,9 @@ import (
 	"go.uber.org/zap/zapcore"
 
 	corev1 "k8s.io/api/core/v1"
-	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/discovery"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -31,7 +28,7 @@ import (
 	firewallv2 "github.com/metal-stack/firewall-controller-manager/api/v2"
 	"github.com/metal-stack/firewall-controller-manager/api/v2/helper"
 
-	firewallv1 "github.com/metal-stack/firewall-controller/api/v1"
+	apihelper "github.com/metal-stack/firewall-controller/api/v1/helper"
 	"github.com/metal-stack/firewall-controller/controllers"
 	"github.com/metal-stack/firewall-controller/pkg/updater"
 	// +kubebuilder:scaffold:imports
@@ -43,19 +40,8 @@ const (
 
 var (
 	setupLog = ctrl.Log.WithName("setup")
-	scheme   = runtime.NewScheme()
 )
 
-func init() {
-	_ = clientgoscheme.AddToScheme(scheme)
-
-	_ = firewallv1.AddToScheme(scheme)
-	_ = firewallv2.AddToScheme(scheme)
-
-	_ = apiextensions.AddToScheme(scheme)
-	// +kubebuilder:scaffold:scheme
-}
-
 func main() {
 	var (
 		logLevel             string
@@ -119,7 +105,7 @@ func main() {
 	}
 
 	seedClient, err := controllerclient.New(seedConfig, controllerclient.Options{
-		Scheme: scheme,
+		Scheme: apihelper.Scheme(),
 	})
 	if err != nil {
 		l.Fatalw("unable to create seed client", "error", err)
@@ -194,7 +180,7 @@ func main() {
 	}
 
 	seedMgr, err := ctrl.NewManager(seedConfig, ctrl.Options{
-		Scheme:             scheme,
+		Scheme:             apihelper.Scheme(),
 		MetricsBindAddress: metricsAddr,
 		Port:               9443,
 		Namespace:          seedNamespace,
@@ -208,15 +194,15 @@ func main() {
 	}
 
 	shootMgr, err := ctrl.NewManager(shootConfig, ctrl.Options{
-		Scheme:             scheme,
+		Scheme:             apihelper.Scheme(),
 		MetricsBindAddress: "0",
 		LeaderElection:     false,
 	})
 	if err != nil {
 		l.Fatalw("unable to create shoot manager", "error", err)
 	}
 
-	shootClient, err := client.New(shootConfig, client.Options{Scheme: scheme})
+	shootClient, err := client.New(shootConfig, client.Options{Scheme: apihelper.Scheme()})
 	if err != nil {
 		l.Fatalw("unable to create shoot client", "error", err)
 	}
@@ -228,7 +214,7 @@ func main() {
 		SeedClient:   seedMgr.GetClient(),
 		ShootClient:  shootClient,
 		Log:          ctrl.Log.WithName("controllers").WithName("Firewall"),
-		Scheme:       scheme,
+		Scheme:       apihelper.Scheme(),
 		Namespace:    seedNamespace,
 		FirewallName: firewallName,
 		Recorder:     shootMgr.GetEventRecorderFor("FirewallController"),
@@ -267,12 +253,14 @@ func main() {
 	}
 
 	if err = (&controllers.FirewallMonitorReconciler{
-		ShootClient:  shootMgr.GetClient(),
-		Log:          ctrl.Log.WithName("controllers").WithName("FirewallMonitorReconciler"),
-		Recorder:     shootMgr.GetEventRecorderFor("FirewallMonitorController"),
-		IDSEnabled:   enableIDS,
-		FirewallName: firewallName,
-		Namespace:    firewallv2.FirewallShootNamespace,
+		ShootClient:        shootMgr.GetClient(),
+		Log:                ctrl.Log.WithName("controllers").WithName("FirewallMonitorReconciler"),
+		Recorder:           shootMgr.GetEventRecorderFor("FirewallMonitorController"),
+		FirewallName:       firewallName,
+		Namespace:          firewallv2.FirewallShootNamespace,
+		IDSEnabled:         enableIDS,
+		SeedNamespace:      seedNamespace,
+		SeedKubeconfigPath: seedKubeconfigPath,
 	}).SetupWithManager(shootMgr); err != nil {
 		l.Fatalw("unable to create firewall monitor controller", "error", err)
 	}
@@ -410,7 +398,7 @@ func controllerMigration(ctx context.Context, log logr.Logger, c client.Client,
 	}
 
 	seed, err := client.New(seedConfig, client.Options{
-		Scheme: scheme,
+		Scheme: apihelper.Scheme(),
 	})
 	if err != nil {
 		return fmt.Errorf("unable to create seed client from migration secret: %w", err)