diff --git a/api/v1/helper/scheme.go b/api/v1/helper/scheme.go new file mode 100644 index 00000000..d94e7c2c --- /dev/null +++ b/api/v1/helper/scheme.go @@ -0,0 +1,29 @@ +package helper + +import ( + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + + firewallv2 "github.com/metal-stack/firewall-controller-manager/api/v2" + firewallv1 "github.com/metal-stack/firewall-controller/api/v1" +) + +var ( + scheme = runtime.NewScheme() +) + +func init() { + _ = clientgoscheme.AddToScheme(scheme) + + _ = firewallv1.AddToScheme(scheme) + _ = firewallv2.AddToScheme(scheme) + + _ = apiextensions.AddToScheme(scheme) + // +kubebuilder:scaffold:scheme +} + +func Scheme() *runtime.Scheme { + return scheme +} diff --git a/controllers/firewall_monitor_controller.go b/controllers/firewall_monitor_controller.go index d823ef7e..9e94ccd1 100644 --- a/controllers/firewall_monitor_controller.go +++ b/controllers/firewall_monitor_controller.go @@ -3,16 +3,22 @@ package controllers import ( "context" "fmt" + "os" "time" "github.com/go-logr/logr" firewallv2 "github.com/metal-stack/firewall-controller-manager/api/v2" firewallv1 "github.com/metal-stack/firewall-controller/api/v1" + apihelper "github.com/metal-stack/firewall-controller/api/v1/helper" "github.com/metal-stack/firewall-controller/pkg/collector" "github.com/metal-stack/firewall-controller/pkg/suricata" "github.com/metal-stack/v" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/clientcmd" + configlatest "k8s.io/client-go/tools/clientcmd/api/latest" + configv1 "k8s.io/client-go/tools/clientcmd/api/v1" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -30,6 +36,9 @@ type FirewallMonitorReconciler struct { FirewallName string Namespace string + SeedNamespace string + SeedKubeconfigPath string + IDSEnabled bool Interval time.Duration } @@ -75,6 +84,10 @@ func (r *FirewallMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, nil } + if err := r.checkSeedEndpoint(ctx, mon); err != nil { + return ctrl.Result{}, err + } + c := collector.NewNFTablesCollector(&r.Log) ruleStats := c.CollectRuleStats() @@ -123,3 +136,98 @@ func (r *FirewallMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ RequeueAfter: r.Interval, }, nil } + +func (r *FirewallMonitorReconciler) checkSeedEndpoint(ctx context.Context, mon *firewallv2.FirewallMonitor) error { + seedURL, ok := mon.Annotations[firewallv2.FirewallSeedURLAnnotation] + if !ok { + return nil + } + + rawKubeconfig, err := os.ReadFile(r.SeedKubeconfigPath) + if err != nil { + return fmt.Errorf("unable to read seed kubeconfig: %w", err) + } + + seedConfig, err := clientcmd.RESTConfigFromKubeConfig(rawKubeconfig) + if err != nil { + return fmt.Errorf("unable to create rest config from seed kubeconfig: %w", err) + } + + if seedConfig.APIPath == seedURL { + return nil + } + + r.Log.Info("seed api url is different in firewall monitor annotation, testing current seed client", "current-url", seedConfig.APIPath, "annotation-url", seedURL) + + clientTest := func(c client.Client) error { + f := &firewallv2.Firewall{ + ObjectMeta: metav1.ObjectMeta{ + Name: mon.Name, + Namespace: r.SeedNamespace, + }, + } + + return c.Get(ctx, client.ObjectKeyFromObject(f), f) + } + + seedClient, err := client.New(seedConfig, client.Options{ + Scheme: apihelper.Scheme(), + }) + if err != nil { + return fmt.Errorf("unable to create seed client from seed kubeconfig: %w", err) + } + + err = clientTest(seedClient) + if err == nil { + r.Log.Info("current seed client seems to work, not taking any further actions") + return nil + } + + r.Log.Error(err, "current seed client seems not to work, attemping seed client update") + + kubeconfig := &configv1.Config{} + err = runtime.DecodeInto(configlatest.Codec, rawKubeconfig, kubeconfig) + if err != nil { + return fmt.Errorf("unable to decode kubeconfig seed kubeconfig: %w", err) + } + + for _, cluster := range kubeconfig.Clusters { + cluster := cluster + cluster.Cluster.Server = seedURL + } + + updatedKubeconfig, err := runtime.Encode(configlatest.Codec, kubeconfig) + if err != nil { + return fmt.Errorf("unable to encode kubeconfig: %w", err) + } + + updatedConfig, err := clientcmd.RESTConfigFromKubeConfig(updatedKubeconfig) + if err != nil { + return fmt.Errorf("unable to create rest config from bytes: %w", err) + } + + newSeedClient, err := client.New(updatedConfig, client.Options{ + Scheme: apihelper.Scheme(), + }) + if err != nil { + return fmt.Errorf("unable to create seed client from updated seed kubeconfig: %w", err) + } + + err = clientTest(newSeedClient) + if err != nil { + return fmt.Errorf("seed client seems broken but seed client with changed api server url also does not appear to work, seed connection lost?") + } + + err = os.WriteFile(r.SeedKubeconfigPath, updatedKubeconfig, 0600) + if err != nil { + return fmt.Errorf("unable to write kubeconfig to destination: %w", err) + } + + r.Log.Info("successfully updating seed client url, restarting controller") + + // not sure if there is a more graceful way to shutdown this controller? + os.Exit(0) + + return nil + +} diff --git a/go.mod b/go.mod index 3cf545bd..f61a3d7d 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/nftables v0.1.0 github.com/ks2211/go-suricata v0.0.0-20200823200910-986ce1470707 - github.com/metal-stack/firewall-controller-manager v0.2.2 + github.com/metal-stack/firewall-controller-manager v0.2.3-0.20230620144431-5175cc6726bb github.com/metal-stack/gardener-extension-provider-metal v0.20.3 github.com/metal-stack/metal-go v0.22.6 github.com/metal-stack/metal-lib v0.11.10 diff --git a/go.sum b/go.sum index d91e1795..18deae12 100644 --- a/go.sum +++ b/go.sum @@ -229,8 +229,8 @@ github.com/mdlayher/netlink v1.7.0 h1:ZNGI4V7i1fJ94DPYtWhI/R85i/Q7ZxnuhUJQcJMood github.com/mdlayher/netlink v1.7.0/go.mod h1:nKO5CSjE/DJjVhk/TNp6vCE1ktVxEA8VEh8drhZzxsQ= github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw= github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc= -github.com/metal-stack/firewall-controller-manager v0.2.2 h1:PM60rnyIb38EVnm4MEm8y2AGeiqwopf8h707qaTJsso= -github.com/metal-stack/firewall-controller-manager v0.2.2/go.mod h1:f4c+yYBQLuNpS5cyGXBj7c0b5TeR770LaElgPgA6DTw= +github.com/metal-stack/firewall-controller-manager v0.2.3-0.20230620144431-5175cc6726bb h1:vYRHLxaNMIc1S376MA9a0i0EFaDbAjTYIGSjrrf43V8= +github.com/metal-stack/firewall-controller-manager v0.2.3-0.20230620144431-5175cc6726bb/go.mod h1:f4c+yYBQLuNpS5cyGXBj7c0b5TeR770LaElgPgA6DTw= github.com/metal-stack/gardener-extension-provider-metal v0.20.3 h1:hhNLjACU2vYbZJFx7XuFXEAZXgXKElq6Bb5FFFUJEiQ= github.com/metal-stack/gardener-extension-provider-metal v0.20.3/go.mod h1:r0SgbEF3au3pJCMmriA3PNaawUd9h3v8msrMt43rGxI= github.com/metal-stack/metal-go v0.22.6 h1:nWjk2VH1B1CWGoupYGxxceblWszTjpD08729riaRyUE= diff --git a/main.go b/main.go index 7755c06e..18534a8d 100644 --- a/main.go +++ b/main.go @@ -15,12 +15,9 @@ import ( "go.uber.org/zap/zapcore" corev1 "k8s.io/api/core/v1" - apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/discovery" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -31,7 +28,7 @@ import ( firewallv2 "github.com/metal-stack/firewall-controller-manager/api/v2" "github.com/metal-stack/firewall-controller-manager/api/v2/helper" - firewallv1 "github.com/metal-stack/firewall-controller/api/v1" + apihelper "github.com/metal-stack/firewall-controller/api/v1/helper" "github.com/metal-stack/firewall-controller/controllers" "github.com/metal-stack/firewall-controller/pkg/updater" // +kubebuilder:scaffold:imports @@ -43,19 +40,8 @@ const ( var ( setupLog = ctrl.Log.WithName("setup") - scheme = runtime.NewScheme() ) -func init() { - _ = clientgoscheme.AddToScheme(scheme) - - _ = firewallv1.AddToScheme(scheme) - _ = firewallv2.AddToScheme(scheme) - - _ = apiextensions.AddToScheme(scheme) - // +kubebuilder:scaffold:scheme -} - func main() { var ( logLevel string @@ -119,7 +105,7 @@ func main() { } seedClient, err := controllerclient.New(seedConfig, controllerclient.Options{ - Scheme: scheme, + Scheme: apihelper.Scheme(), }) if err != nil { l.Fatalw("unable to create seed client", "error", err) @@ -194,7 +180,7 @@ func main() { } seedMgr, err := ctrl.NewManager(seedConfig, ctrl.Options{ - Scheme: scheme, + Scheme: apihelper.Scheme(), MetricsBindAddress: metricsAddr, Port: 9443, Namespace: seedNamespace, @@ -208,7 +194,7 @@ func main() { } shootMgr, err := ctrl.NewManager(shootConfig, ctrl.Options{ - Scheme: scheme, + Scheme: apihelper.Scheme(), MetricsBindAddress: "0", LeaderElection: false, }) @@ -216,7 +202,7 @@ func main() { l.Fatalw("unable to create shoot manager", "error", err) } - shootClient, err := client.New(shootConfig, client.Options{Scheme: scheme}) + shootClient, err := client.New(shootConfig, client.Options{Scheme: apihelper.Scheme()}) if err != nil { l.Fatalw("unable to create shoot client", "error", err) } @@ -228,7 +214,7 @@ func main() { SeedClient: seedMgr.GetClient(), ShootClient: shootClient, Log: ctrl.Log.WithName("controllers").WithName("Firewall"), - Scheme: scheme, + Scheme: apihelper.Scheme(), Namespace: seedNamespace, FirewallName: firewallName, Recorder: shootMgr.GetEventRecorderFor("FirewallController"), @@ -267,12 +253,14 @@ func main() { } if err = (&controllers.FirewallMonitorReconciler{ - ShootClient: shootMgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("FirewallMonitorReconciler"), - Recorder: shootMgr.GetEventRecorderFor("FirewallMonitorController"), - IDSEnabled: enableIDS, - FirewallName: firewallName, - Namespace: firewallv2.FirewallShootNamespace, + ShootClient: shootMgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("FirewallMonitorReconciler"), + Recorder: shootMgr.GetEventRecorderFor("FirewallMonitorController"), + FirewallName: firewallName, + Namespace: firewallv2.FirewallShootNamespace, + IDSEnabled: enableIDS, + SeedNamespace: seedNamespace, + SeedKubeconfigPath: seedKubeconfigPath, }).SetupWithManager(shootMgr); err != nil { l.Fatalw("unable to create firewall monitor controller", "error", err) } @@ -410,7 +398,7 @@ func controllerMigration(ctx context.Context, log logr.Logger, c client.Client, } seed, err := client.New(seedConfig, client.Options{ - Scheme: scheme, + Scheme: apihelper.Scheme(), }) if err != nil { return fmt.Errorf("unable to create seed client from migration secret: %w", err)