From cc3a9039556bbf62af9909f38020c419958ce524 Mon Sep 17 00:00:00 2001 From: Johannes Rauh Date: Tue, 19 Nov 2024 15:15:05 +0100 Subject: [PATCH 01/13] Add keys for prometheus config --- pkg/schema/v1/config.go | 3 +++ schema/mysql/schema.sql | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/schema/v1/config.go b/pkg/schema/v1/config.go index 09dcac6..239504b 100644 --- a/pkg/schema/v1/config.go +++ b/pkg/schema/v1/config.go @@ -18,4 +18,7 @@ const ( ConfigKeyNotificationsPassword ConfigKey = "notifications.password" ConfigKeyNotificationsUrl ConfigKey = "notifications.url" ConfigKeyNotificationsKubernetesWebUrl ConfigKey = "notifications.kubernetes_web_url" + ConfigKeyPrometheusUrl ConfigKey = "prometheus.url" + ConfigKeyPrometheusUsername ConfigKey = "prometheus.username" + ConfigKeyPrometheusPassword ConfigKey = "prometheus.password" ) diff --git a/schema/mysql/schema.sql b/schema/mysql/schema.sql index 03e1677..e9562ef 100644 --- a/schema/mysql/schema.sql +++ b/schema/mysql/schema.sql @@ -1018,11 +1018,13 @@ CREATE TABLE kubernetes_instance ( CREATE TABLE config ( cluster_uuid binary(16) NOT NULL, `key` enum( - 'notifications.url', 'notifications.username', 'notifications.password', - 'notifications.kubernetes_web_url' - ) COLLATE utf8mb4_unicode_ci NOT NULL, + 'notifications.source_id', + 'prometheus.url', + 'prometheus.username', + 'prometheus.password' + ) COLLATE utf8mb4_unicode_ci NOT NULL, value varchar(255) NOT NULL, locked enum('n', 'y') COLLATE utf8mb4_unicode_ci NOT NULL, From 2a6bfca7f78be2dd71c1625e37441c633df929c8 Mon Sep 17 00:00:00 2001 From: Johannes Rauh Date: Tue, 19 Nov 2024 15:15:38 +0100 Subject: [PATCH 02/13] Add basic auth --- internal/basic_auth.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 internal/basic_auth.go diff --git a/internal/basic_auth.go b/internal/basic_auth.go new file mode 100644 index 0000000..9a5c96b --- /dev/null +++ b/internal/basic_auth.go @@ -0,0 +1,18 @@ +package internal + +import ( + "net/http" +) + +// BasicAuthTransport is a http.RoundTripper that authenticates all requests using HTTP Basic Authentication. +type BasicAuthTransport struct { + Username string + Password string +} + +// RoundTrip executes a single HTTP transaction with the basic auth credentials. +func (rt *BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.SetBasicAuth(rt.Username, rt.Password) + + return http.DefaultTransport.RoundTrip(req) +} From 423ff004b76bd5ace4d0de39be9a982de8ef1a15 Mon Sep 17 00:00:00 2001 From: Johannes Rauh Date: Tue, 19 Nov 2024 15:16:12 +0100 Subject: [PATCH 03/13] Sync prometheus config to database --- cmd/icinga-kubernetes/main.go | 31 ++++++- pkg/metrics/config.go | 156 +++++++++++++++++++++++++++++++++- 2 files changed, 184 insertions(+), 3 deletions(-) diff --git a/cmd/icinga-kubernetes/main.go b/cmd/icinga-kubernetes/main.go index d9e2cd8..1095813 100644 --- a/cmd/icinga-kubernetes/main.go +++ b/cmd/icinga-kubernetes/main.go @@ -36,6 +36,7 @@ import ( "k8s.io/client-go/kubernetes" kclientcmd "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" + "net/http" "os" "strings" "sync" @@ -298,10 +299,36 @@ func main() { return SyncServicePods(ctx, kdb, factory.Core().V1().Services(), factory.Core().V1().Pods()) }) + err = metrics.SyncPrometheusConfig(ctx, db, &cfg.Prometheus) + if err != nil { + klog.Error(errors.Wrap(err, "cannot sync prometheus config")) + } + + if cfg.Prometheus.Url == "" { + if cfg.Prometheus.Url == "" { + err = metrics.AutoDetectPrometheus(ctx, clientset, &cfg.Prometheus) + if err != nil { + klog.Error(errors.Wrap(err, "cannot auto-detect prometheus")) + } + } + } + if cfg.Prometheus.Url != "" { - promClient, err := promapi.NewClient(promapi.Config{Address: cfg.Prometheus.Url}) + var basicAuthTransport http.RoundTripper + + if cfg.Prometheus.Username != "" && cfg.Prometheus.Password != "" { + basicAuthTransport = &internal.BasicAuthTransport{ + Username: cfg.Prometheus.Username, + Password: cfg.Prometheus.Password, + } + } + + promClient, err := promapi.NewClient(promapi.Config{ + Address: cfg.Prometheus.Url, + RoundTripper: basicAuthTransport, + }) if err != nil { - klog.Fatal(errors.Wrap(err, "error creating promClient")) + klog.Fatal(errors.Wrap(err, "error creating Prometheus client")) } promApiClient := promv1.NewAPI(promClient) diff --git a/pkg/metrics/config.go b/pkg/metrics/config.go index 4b21892..b6aafb7 100644 --- a/pkg/metrics/config.go +++ b/pkg/metrics/config.go @@ -1,11 +1,165 @@ package metrics +import ( + "context" + "database/sql" + "fmt" + "github.com/icinga/icinga-go-library/database" + "github.com/icinga/icinga-go-library/types" + schemav1 "github.com/icinga/icinga-kubernetes/pkg/schema/v1" + "github.com/pkg/errors" + kmetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "strings" +) + // PrometheusConfig defines Prometheus configuration. type PrometheusConfig struct { - Url string `yaml:"url"` + Url string `yaml:"url"` + Username string `yaml:"username"` + Password string `yaml:"password"` } // Validate checks constraints in the supplied Prometheus configuration and returns an error if they are violated. func (c *PrometheusConfig) Validate() error { + if c.Url == "" && (c.Username != "" || c.Password != "") { + return errors.New("credentials cannot be provided without a URL") + } + if (c.Username == "") != (c.Password == "") { + return errors.New("both username and password must be provided") + } + return nil +} + +func SyncPrometheusConfig(ctx context.Context, db *database.DB, config *PrometheusConfig) error { + _true := types.Bool{Bool: true, Valid: true} + + var configPairs []*schemav1.Config + var deleteKeys []schemav1.ConfigKey + + tx, err := db.BeginTxx(ctx, &sql.TxOptions{Isolation: sql.LevelSerializable}) + if err != nil { + return errors.Wrap(err, "cannot start transaction") + } + + if config.Url != "" { + configPairs = append(configPairs, &schemav1.Config{Key: schemav1.ConfigKeyPrometheusUrl, Value: config.Url, Locked: _true}) + + if config.Username != "" { + configPairs = append(configPairs, &schemav1.Config{Key: schemav1.ConfigKeyPrometheusUsername, Value: config.Username, Locked: _true}) + configPairs = append(configPairs, &schemav1.Config{Key: schemav1.ConfigKeyPrometheusPassword, Value: config.Password, Locked: _true}) + } else { + deleteKeys = append(deleteKeys, schemav1.ConfigKeyPrometheusUsername) + deleteKeys = append(deleteKeys, schemav1.ConfigKeyPrometheusPassword) + } + } else { + deleteKeys, err = cleanupKeys(ctx, db) + if err != nil { + if err := tx.Rollback(); err != nil { + return errors.Wrap(err, "cannot rollback transaction") + } + return errors.Wrap(err, "cannot cleanup Prometheus configuration") + } + } + + if len(configPairs) > 0 { + upsertStmt, _ := db.BuildUpsertStmt(&schemav1.Config{}) + + if _, err := tx.NamedExecContext(ctx, upsertStmt, configPairs); err != nil { + if err := tx.Rollback(); err != nil { + return errors.Wrap(err, "cannot rollback transaction") + } + return errors.Wrap(err, "cannot upsert Prometheus configuration") + } + } + + if len(deleteKeys) > 0 { + deleteStmt := fmt.Sprintf( + `DELETE FROM %s WHERE %s = (?)`, + database.TableName(&schemav1.Config{}), + "`key`", + ) + + for _, key := range deleteKeys { + if _, err := tx.ExecContext(ctx, deleteStmt, key); err != nil { + if err := tx.Rollback(); err != nil { + return errors.Wrap(err, "cannot rollback transaction") + } + return errors.Wrap(err, "cannot delete Prometheus credentials") + } + } + } + + if err := tx.Commit(); err != nil { + return errors.Wrap(err, "cannot commit transaction") + } + + return nil +} + +func cleanupKeys(ctx context.Context, db *database.DB) ([]schemav1.ConfigKey, error) { + var dbConfig []*schemav1.Config + if err := db.SelectContext(ctx, &dbConfig, db.BuildSelectStmt(&schemav1.Config{}, &schemav1.Config{})); err != nil { + return nil, errors.Wrap(err, "cannot retrieve Prometheus configuration") + } + + var deleteKeys []schemav1.ConfigKey + + for _, c := range dbConfig { + if c.Locked.Bool { + switch c.Key { + case schemav1.ConfigKeyPrometheusUrl: + deleteKeys = append(deleteKeys, schemav1.ConfigKeyPrometheusUrl) + case schemav1.ConfigKeyPrometheusUsername: + deleteKeys = append(deleteKeys, schemav1.ConfigKeyPrometheusUsername) + case schemav1.ConfigKeyPrometheusPassword: + deleteKeys = append(deleteKeys, schemav1.ConfigKeyPrometheusPassword) + } + } + } + + return deleteKeys, nil +} + +func AutoDetectPrometheus(ctx context.Context, clientset *kubernetes.Clientset, config *PrometheusConfig) error { + services, err := clientset.CoreV1().Services("monitoring").List(ctx, kmetav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=prometheus", + }) + if err != nil { + return errors.Wrap(err, "cannot list Prometheus services") + } + + if len(services.Items) == 0 { + return errors.New("no Prometheus service found") + } + + var ip string + var port int32 + + // Check if we are running in a Kubernetes cluster. If so, use the + // service's ClusterIP. Otherwise, use the API Server's IP and NodePort. + if _, err = rest.InClusterConfig(); err == nil { + for _, service := range services.Items { + if service.Spec.Type == "ClusterIP" { + ip = services.Items[0].Spec.ClusterIP + port = services.Items[0].Spec.Ports[0].Port + + break + } + } + } else if errors.Is(err, rest.ErrNotInCluster) { + for _, service := range services.Items { + if service.Spec.Type == "NodePort" { + ip = strings.Split(clientset.RESTClient().Get().URL().Host, ":")[0] + port = service.Spec.Ports[0].NodePort + + break + } + } + } + + config.Url = fmt.Sprintf("http://%s:%d", ip, port) + return nil } From 9a4489ef023805b31b4c79ee288fa1d924c6d551 Mon Sep 17 00:00:00 2001 From: Johannes Rauh Date: Wed, 18 Dec 2024 11:44:10 +0100 Subject: [PATCH 04/13] Move BasicAuthTransport to package 'com' --- .../basic_auth.go => pkg/com/basic_auth_transport.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) rename internal/basic_auth.go => pkg/com/basic_auth_transport.go (59%) diff --git a/internal/basic_auth.go b/pkg/com/basic_auth_transport.go similarity index 59% rename from internal/basic_auth.go rename to pkg/com/basic_auth_transport.go index 9a5c96b..c609d25 100644 --- a/internal/basic_auth.go +++ b/pkg/com/basic_auth_transport.go @@ -1,4 +1,4 @@ -package internal +package com import ( "net/http" @@ -6,13 +6,14 @@ import ( // BasicAuthTransport is a http.RoundTripper that authenticates all requests using HTTP Basic Authentication. type BasicAuthTransport struct { + http.RoundTripper Username string Password string } // RoundTrip executes a single HTTP transaction with the basic auth credentials. -func (rt *BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) { - req.SetBasicAuth(rt.Username, rt.Password) +func (t *BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.SetBasicAuth(t.Username, t.Password) - return http.DefaultTransport.RoundTrip(req) + return t.RoundTripper.RoundTrip(req) } From 2802d14ac9b6d584375312bde26ae4b42735062c Mon Sep 17 00:00:00 2001 From: Johannes Rauh Date: Wed, 18 Dec 2024 11:45:52 +0100 Subject: [PATCH 05/13] Use com.BasicAuthTransport --- cmd/icinga-kubernetes/main.go | 2 +- pkg/notifications/client.go | 19 ++++--------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/cmd/icinga-kubernetes/main.go b/cmd/icinga-kubernetes/main.go index 1095813..fa5f6f6 100644 --- a/cmd/icinga-kubernetes/main.go +++ b/cmd/icinga-kubernetes/main.go @@ -317,7 +317,7 @@ func main() { var basicAuthTransport http.RoundTripper if cfg.Prometheus.Username != "" && cfg.Prometheus.Password != "" { - basicAuthTransport = &internal.BasicAuthTransport{ + basicAuthTransport = &com.BasicAuthTransport{ Username: cfg.Prometheus.Username, Password: cfg.Prometheus.Password, } diff --git a/pkg/notifications/client.go b/pkg/notifications/client.go index 8c6b31e..cc00d79 100644 --- a/pkg/notifications/client.go +++ b/pkg/notifications/client.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "github.com/icinga/icinga-kubernetes/pkg/com" "github.com/pkg/errors" "io" "k8s.io/klog/v2" @@ -31,10 +32,10 @@ func NewClient(name string, config Config) (*Client, error) { return &Client{ client: http.Client{ - Transport: &basicAuthTransport{ + Transport: &com.BasicAuthTransport{ RoundTripper: http.DefaultTransport, - username: config.Username, - password: config.Password, + Username: config.Username, + Password: config.Password, }, }, userAgent: name, @@ -93,15 +94,3 @@ func (c *Client) Stream(ctx context.Context, entities <-chan any) error { } } } - -type basicAuthTransport struct { - http.RoundTripper - username string - password string -} - -func (t *basicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) { - req.SetBasicAuth(t.username, t.password) - - return t.RoundTripper.RoundTrip(req) -} From e284d40071efc2eab443c00867a81aaf15a02fbf Mon Sep 17 00:00:00 2001 From: Johannes Rauh Date: Wed, 18 Dec 2024 11:47:45 +0100 Subject: [PATCH 06/13] Remove double if --- cmd/icinga-kubernetes/main.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/icinga-kubernetes/main.go b/cmd/icinga-kubernetes/main.go index fa5f6f6..824db67 100644 --- a/cmd/icinga-kubernetes/main.go +++ b/cmd/icinga-kubernetes/main.go @@ -305,11 +305,9 @@ func main() { } if cfg.Prometheus.Url == "" { - if cfg.Prometheus.Url == "" { - err = metrics.AutoDetectPrometheus(ctx, clientset, &cfg.Prometheus) - if err != nil { - klog.Error(errors.Wrap(err, "cannot auto-detect prometheus")) - } + err = metrics.AutoDetectPrometheus(ctx, clientset, &cfg.Prometheus) + if err != nil { + klog.Error(errors.Wrap(err, "cannot auto-detect prometheus")) } } From df3e566d823e8f283388f2164e558e2287f82072 Mon Sep 17 00:00:00 2001 From: Johannes Rauh Date: Wed, 18 Dec 2024 13:21:21 +0100 Subject: [PATCH 07/13] Fix error message --- internal/notifications.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/notifications.go b/internal/notifications.go index e95089d..8948324 100644 --- a/internal/notifications.go +++ b/internal/notifications.go @@ -103,7 +103,7 @@ func SyncNotificationsConfig(ctx context.Context, db *database.DB, config *notif return nil }) if err != nil { - return errors.Wrap(err, "cannot upsert Icinga Notifications config") + return errors.Wrap(err, "cannot retrieve Icinga Notifications config") } } From 92ce778bc3d452ee6f07e5bdcf3e9c72a742db5f Mon Sep 17 00:00:00 2001 From: Johannes Rauh Date: Wed, 18 Dec 2024 13:44:31 +0100 Subject: [PATCH 08/13] Use transaction to select from database --- internal/notifications.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/notifications.go b/internal/notifications.go index 8948324..294db4c 100644 --- a/internal/notifications.go +++ b/internal/notifications.go @@ -77,7 +77,7 @@ func SyncNotificationsConfig(ctx context.Context, db *database.DB, config *notif return errors.Wrap(err, "cannot delete Icinga Notifications config") } - rows, err := db.QueryxContext(ctx, db.BuildSelectStmt(&schemav1.Config{}, &schemav1.Config{})) + rows, err := tx.QueryxContext(ctx, db.BuildSelectStmt(&schemav1.Config{}, &schemav1.Config{})) if err != nil { return errors.Wrap(err, "cannot fetch Icinga Notifications config from DB") } From 80dee353bf55a47f0ee36f90d58bd62689fba514 Mon Sep 17 00:00:00 2001 From: Johannes Rauh Date: Wed, 18 Dec 2024 14:06:20 +0100 Subject: [PATCH 09/13] Sync Prometheus config like for Notifications --- cmd/icinga-kubernetes/main.go | 10 ++- internal/prometheus.go | 150 ++++++++++++++++++++++++++++++++++ pkg/metrics/config.go | 142 -------------------------------- 3 files changed, 156 insertions(+), 146 deletions(-) create mode 100644 internal/prometheus.go diff --git a/cmd/icinga-kubernetes/main.go b/cmd/icinga-kubernetes/main.go index 824db67..4003135 100644 --- a/cmd/icinga-kubernetes/main.go +++ b/cmd/icinga-kubernetes/main.go @@ -15,6 +15,7 @@ import ( "github.com/icinga/icinga-kubernetes/internal" cachev1 "github.com/icinga/icinga-kubernetes/internal/cache/v1" "github.com/icinga/icinga-kubernetes/pkg/cluster" + "github.com/icinga/icinga-kubernetes/pkg/com" "github.com/icinga/icinga-kubernetes/pkg/daemon" kdatabase "github.com/icinga/icinga-kubernetes/pkg/database" "github.com/icinga/icinga-kubernetes/pkg/metrics" @@ -299,13 +300,13 @@ func main() { return SyncServicePods(ctx, kdb, factory.Core().V1().Services(), factory.Core().V1().Pods()) }) - err = metrics.SyncPrometheusConfig(ctx, db, &cfg.Prometheus) + err = internal.SyncPrometheusConfig(ctx, db, &cfg.Prometheus) if err != nil { klog.Error(errors.Wrap(err, "cannot sync prometheus config")) } if cfg.Prometheus.Url == "" { - err = metrics.AutoDetectPrometheus(ctx, clientset, &cfg.Prometheus) + err = internal.AutoDetectPrometheus(ctx, clientset, &cfg.Prometheus) if err != nil { klog.Error(errors.Wrap(err, "cannot auto-detect prometheus")) } @@ -316,8 +317,9 @@ func main() { if cfg.Prometheus.Username != "" && cfg.Prometheus.Password != "" { basicAuthTransport = &com.BasicAuthTransport{ - Username: cfg.Prometheus.Username, - Password: cfg.Prometheus.Password, + RoundTripper: http.DefaultTransport, + Username: cfg.Prometheus.Username, + Password: cfg.Prometheus.Password, } } diff --git a/internal/prometheus.go b/internal/prometheus.go new file mode 100644 index 0000000..ff2c9f4 --- /dev/null +++ b/internal/prometheus.go @@ -0,0 +1,150 @@ +package internal + +import ( + "context" + "fmt" + "github.com/icinga/icinga-go-library/database" + "github.com/icinga/icinga-go-library/types" + "github.com/icinga/icinga-kubernetes/pkg/metrics" + schemav1 "github.com/icinga/icinga-kubernetes/pkg/schema/v1" + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + kmetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "strings" +) + +func SyncPrometheusConfig(ctx context.Context, db *database.DB, config *metrics.PrometheusConfig) error { + _true := types.Bool{Bool: true, Valid: true} + + if config.Url != "" { + toDb := []schemav1.Config{ + {Key: schemav1.ConfigKeyPrometheusUrl, Value: config.Url, Locked: _true}, + } + + if config.Username != "" { + toDb = append( + toDb, + schemav1.Config{Key: schemav1.ConfigKeyPrometheusUsername, Value: config.Username, Locked: _true}, + schemav1.Config{Key: schemav1.ConfigKeyPrometheusPassword, Value: config.Password, Locked: _true}, + ) + } + + err := db.ExecTx(ctx, func(ctx context.Context, tx *sqlx.Tx) error { + if _, err := tx.ExecContext( + ctx, + fmt.Sprintf( + `DELETE FROM "%s" WHERE "key" LIKE ? AND "locked" = ?`, + database.TableName(&schemav1.Config{}), + ), + `prometheus.%`, + _true, + ); err != nil { + return errors.Wrap(err, "cannot delete Prometheus config") + } + + stmt, _ := db.BuildInsertStmt(schemav1.Config{}) + if _, err := tx.NamedExecContext(ctx, stmt, toDb); err != nil { + return errors.Wrap(err, "cannot insert Prometheus config") + } + + return nil + }) + if err != nil { + return errors.Wrap(err, "cannot upsert Prometheus config") + } + } else { + err := db.ExecTx(ctx, func(ctx context.Context, tx *sqlx.Tx) error { + if _, err := tx.ExecContext( + ctx, + fmt.Sprintf( + `DELETE FROM "%s" WHERE "key" LIKE ? AND "locked" = ?`, + database.TableName(&schemav1.Config{}), + ), + `prometheus.%`, + _true, + ); err != nil { + return errors.Wrap(err, "cannot delete Prometheus config") + } + + rows, err := tx.QueryxContext(ctx, db.BuildSelectStmt(&schemav1.Config{}, &schemav1.Config{})) + if err != nil { + return errors.Wrap(err, "cannot fetch Prometheus config from DB") + } + + for rows.Next() { + var r schemav1.Config + if err := rows.StructScan(&r); err != nil { + return errors.Wrap(err, "cannot fetch Prometheus config from DB") + } + + switch r.Key { + case schemav1.ConfigKeyPrometheusUrl: + config.Url = r.Value + case schemav1.ConfigKeyPrometheusUsername: + config.Username = r.Value + case schemav1.ConfigKeyPrometheusPassword: + config.Password = r.Value + } + } + + return nil + }) + if err != nil { + return errors.Wrap(err, "cannot retrieve Prometheus config") + } + } + + return nil +} + +// AutoDetectPrometheus tries to auto-detect the Prometheus service in the monitoring namespace and +// if found sets the URL in the supplied Prometheus configuration. The first service with the label +// "app.kubernetes.io/name=prometheus" is used. Until now the ServiceTypes ClusterIP and NodePort are supported. +func AutoDetectPrometheus(ctx context.Context, clientset *kubernetes.Clientset, config *metrics.PrometheusConfig) error { + services, err := clientset.CoreV1().Services("monitoring").List(ctx, kmetav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=prometheus", + }) + if err != nil { + return errors.Wrap(err, "cannot list Prometheus services") + } + + if len(services.Items) == 0 { + return errors.New("no Prometheus service found") + } + + var ip string + var port int32 + + // Check if we are running in a Kubernetes cluster. If so, use the + // service's ClusterIP. Otherwise, use the API Server's IP and NodePort. + if _, err = rest.InClusterConfig(); err == nil { + for _, service := range services.Items { + if service.Spec.Type == v1.ServiceTypeClusterIP { + ip = service.Spec.ClusterIP + port = service.Spec.Ports[0].Port + + break + } + } + } else if errors.Is(err, rest.ErrNotInCluster) { + for _, service := range services.Items { + if service.Spec.Type == v1.ServiceTypeNodePort { + ip = strings.Split(clientset.RESTClient().Get().URL().Host, ":")[0] + port = service.Spec.Ports[0].NodePort + + break + } + } + } + + if ip == "" { + + } + + config.Url = fmt.Sprintf("http://%s:%d", ip, port) + + return nil +} diff --git a/pkg/metrics/config.go b/pkg/metrics/config.go index b6aafb7..41fae6b 100644 --- a/pkg/metrics/config.go +++ b/pkg/metrics/config.go @@ -1,17 +1,7 @@ package metrics import ( - "context" - "database/sql" - "fmt" - "github.com/icinga/icinga-go-library/database" - "github.com/icinga/icinga-go-library/types" - schemav1 "github.com/icinga/icinga-kubernetes/pkg/schema/v1" "github.com/pkg/errors" - kmetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "strings" ) // PrometheusConfig defines Prometheus configuration. @@ -31,135 +21,3 @@ func (c *PrometheusConfig) Validate() error { } return nil } - -func SyncPrometheusConfig(ctx context.Context, db *database.DB, config *PrometheusConfig) error { - _true := types.Bool{Bool: true, Valid: true} - - var configPairs []*schemav1.Config - var deleteKeys []schemav1.ConfigKey - - tx, err := db.BeginTxx(ctx, &sql.TxOptions{Isolation: sql.LevelSerializable}) - if err != nil { - return errors.Wrap(err, "cannot start transaction") - } - - if config.Url != "" { - configPairs = append(configPairs, &schemav1.Config{Key: schemav1.ConfigKeyPrometheusUrl, Value: config.Url, Locked: _true}) - - if config.Username != "" { - configPairs = append(configPairs, &schemav1.Config{Key: schemav1.ConfigKeyPrometheusUsername, Value: config.Username, Locked: _true}) - configPairs = append(configPairs, &schemav1.Config{Key: schemav1.ConfigKeyPrometheusPassword, Value: config.Password, Locked: _true}) - } else { - deleteKeys = append(deleteKeys, schemav1.ConfigKeyPrometheusUsername) - deleteKeys = append(deleteKeys, schemav1.ConfigKeyPrometheusPassword) - } - } else { - deleteKeys, err = cleanupKeys(ctx, db) - if err != nil { - if err := tx.Rollback(); err != nil { - return errors.Wrap(err, "cannot rollback transaction") - } - return errors.Wrap(err, "cannot cleanup Prometheus configuration") - } - } - - if len(configPairs) > 0 { - upsertStmt, _ := db.BuildUpsertStmt(&schemav1.Config{}) - - if _, err := tx.NamedExecContext(ctx, upsertStmt, configPairs); err != nil { - if err := tx.Rollback(); err != nil { - return errors.Wrap(err, "cannot rollback transaction") - } - return errors.Wrap(err, "cannot upsert Prometheus configuration") - } - } - - if len(deleteKeys) > 0 { - deleteStmt := fmt.Sprintf( - `DELETE FROM %s WHERE %s = (?)`, - database.TableName(&schemav1.Config{}), - "`key`", - ) - - for _, key := range deleteKeys { - if _, err := tx.ExecContext(ctx, deleteStmt, key); err != nil { - if err := tx.Rollback(); err != nil { - return errors.Wrap(err, "cannot rollback transaction") - } - return errors.Wrap(err, "cannot delete Prometheus credentials") - } - } - } - - if err := tx.Commit(); err != nil { - return errors.Wrap(err, "cannot commit transaction") - } - - return nil -} - -func cleanupKeys(ctx context.Context, db *database.DB) ([]schemav1.ConfigKey, error) { - var dbConfig []*schemav1.Config - if err := db.SelectContext(ctx, &dbConfig, db.BuildSelectStmt(&schemav1.Config{}, &schemav1.Config{})); err != nil { - return nil, errors.Wrap(err, "cannot retrieve Prometheus configuration") - } - - var deleteKeys []schemav1.ConfigKey - - for _, c := range dbConfig { - if c.Locked.Bool { - switch c.Key { - case schemav1.ConfigKeyPrometheusUrl: - deleteKeys = append(deleteKeys, schemav1.ConfigKeyPrometheusUrl) - case schemav1.ConfigKeyPrometheusUsername: - deleteKeys = append(deleteKeys, schemav1.ConfigKeyPrometheusUsername) - case schemav1.ConfigKeyPrometheusPassword: - deleteKeys = append(deleteKeys, schemav1.ConfigKeyPrometheusPassword) - } - } - } - - return deleteKeys, nil -} - -func AutoDetectPrometheus(ctx context.Context, clientset *kubernetes.Clientset, config *PrometheusConfig) error { - services, err := clientset.CoreV1().Services("monitoring").List(ctx, kmetav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name=prometheus", - }) - if err != nil { - return errors.Wrap(err, "cannot list Prometheus services") - } - - if len(services.Items) == 0 { - return errors.New("no Prometheus service found") - } - - var ip string - var port int32 - - // Check if we are running in a Kubernetes cluster. If so, use the - // service's ClusterIP. Otherwise, use the API Server's IP and NodePort. - if _, err = rest.InClusterConfig(); err == nil { - for _, service := range services.Items { - if service.Spec.Type == "ClusterIP" { - ip = services.Items[0].Spec.ClusterIP - port = services.Items[0].Spec.Ports[0].Port - - break - } - } - } else if errors.Is(err, rest.ErrNotInCluster) { - for _, service := range services.Items { - if service.Spec.Type == "NodePort" { - ip = strings.Split(clientset.RESTClient().Get().URL().Host, ":")[0] - port = service.Spec.Ports[0].NodePort - - break - } - } - } - - config.Url = fmt.Sprintf("http://%s:%d", ip, port) - - return nil -} From 7a4bfc4c738be70a81d82a7d74349b200d81a447 Mon Sep 17 00:00:00 2001 From: Johannes Rauh Date: Wed, 18 Dec 2024 14:48:57 +0100 Subject: [PATCH 10/13] Add multi cluster support --- cmd/icinga-kubernetes/main.go | 2 +- internal/prometheus.go | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/icinga-kubernetes/main.go b/cmd/icinga-kubernetes/main.go index 4003135..b6f9eaa 100644 --- a/cmd/icinga-kubernetes/main.go +++ b/cmd/icinga-kubernetes/main.go @@ -300,7 +300,7 @@ func main() { return SyncServicePods(ctx, kdb, factory.Core().V1().Services(), factory.Core().V1().Pods()) }) - err = internal.SyncPrometheusConfig(ctx, db, &cfg.Prometheus) + err = internal.SyncPrometheusConfig(ctx, db, &cfg.Prometheus, clusterInstance.Uuid) if err != nil { klog.Error(errors.Wrap(err, "cannot sync prometheus config")) } diff --git a/internal/prometheus.go b/internal/prometheus.go index ff2c9f4..90e1ee1 100644 --- a/internal/prometheus.go +++ b/internal/prometheus.go @@ -16,19 +16,19 @@ import ( "strings" ) -func SyncPrometheusConfig(ctx context.Context, db *database.DB, config *metrics.PrometheusConfig) error { +func SyncPrometheusConfig(ctx context.Context, db *database.DB, config *metrics.PrometheusConfig, clusterUuid types.UUID) error { _true := types.Bool{Bool: true, Valid: true} if config.Url != "" { toDb := []schemav1.Config{ - {Key: schemav1.ConfigKeyPrometheusUrl, Value: config.Url, Locked: _true}, + {ClusterUuid: clusterUuid, Key: schemav1.ConfigKeyPrometheusUrl, Value: config.Url, Locked: _true}, } if config.Username != "" { toDb = append( toDb, - schemav1.Config{Key: schemav1.ConfigKeyPrometheusUsername, Value: config.Username, Locked: _true}, - schemav1.Config{Key: schemav1.ConfigKeyPrometheusPassword, Value: config.Password, Locked: _true}, + schemav1.Config{ClusterUuid: clusterUuid, Key: schemav1.ConfigKeyPrometheusUsername, Value: config.Username, Locked: _true}, + schemav1.Config{ClusterUuid: clusterUuid, Key: schemav1.ConfigKeyPrometheusPassword, Value: config.Password, Locked: _true}, ) } @@ -36,9 +36,10 @@ func SyncPrometheusConfig(ctx context.Context, db *database.DB, config *metrics. if _, err := tx.ExecContext( ctx, fmt.Sprintf( - `DELETE FROM "%s" WHERE "key" LIKE ? AND "locked" = ?`, + `DELETE FROM "%s" WHERE "cluster_uuid" = ? AND "key" LIKE ? AND "locked" = ?`, database.TableName(&schemav1.Config{}), ), + clusterUuid, `prometheus.%`, _true, ); err != nil { @@ -60,9 +61,10 @@ func SyncPrometheusConfig(ctx context.Context, db *database.DB, config *metrics. if _, err := tx.ExecContext( ctx, fmt.Sprintf( - `DELETE FROM "%s" WHERE "key" LIKE ? AND "locked" = ?`, + `DELETE FROM "%s" WHERE "cluster_uuid" = ? AND "key" LIKE ? AND "locked" = ?`, database.TableName(&schemav1.Config{}), ), + clusterUuid, `prometheus.%`, _true, ); err != nil { From 816fb9c118f124d53f99b2498048806be8ba3030 Mon Sep 17 00:00:00 2001 From: Johannes Rauh Date: Mon, 30 Dec 2024 15:28:31 +0100 Subject: [PATCH 11/13] Add keys to config table --- schema/mysql/schema.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/schema/mysql/schema.sql b/schema/mysql/schema.sql index e9562ef..fa06dcf 100644 --- a/schema/mysql/schema.sql +++ b/schema/mysql/schema.sql @@ -1020,6 +1020,8 @@ CREATE TABLE config ( `key` enum( 'notifications.username', 'notifications.password', + 'notifications.url', + 'notifications.kubernetes_web_url', 'notifications.source_id', 'prometheus.url', 'prometheus.username', From 031e0f87a60be7f0c04569d3fbe87b3cbded8b18 Mon Sep 17 00:00:00 2001 From: Johannes Rauh Date: Tue, 14 Jan 2025 10:23:53 +0100 Subject: [PATCH 12/13] Fix requested changes --- internal/prometheus.go | 2 +- pkg/metrics/config.go | 3 --- schema/mysql/schema.sql | 3 +-- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/internal/prometheus.go b/internal/prometheus.go index 90e1ee1..35391f7 100644 --- a/internal/prometheus.go +++ b/internal/prometheus.go @@ -143,7 +143,7 @@ func AutoDetectPrometheus(ctx context.Context, clientset *kubernetes.Clientset, } if ip == "" { - + return errors.New("no Prometheus found") } config.Url = fmt.Sprintf("http://%s:%d", ip, port) diff --git a/pkg/metrics/config.go b/pkg/metrics/config.go index 41fae6b..a2e64e9 100644 --- a/pkg/metrics/config.go +++ b/pkg/metrics/config.go @@ -13,9 +13,6 @@ type PrometheusConfig struct { // Validate checks constraints in the supplied Prometheus configuration and returns an error if they are violated. func (c *PrometheusConfig) Validate() error { - if c.Url == "" && (c.Username != "" || c.Password != "") { - return errors.New("credentials cannot be provided without a URL") - } if (c.Username == "") != (c.Password == "") { return errors.New("both username and password must be provided") } diff --git a/schema/mysql/schema.sql b/schema/mysql/schema.sql index fa06dcf..971c952 100644 --- a/schema/mysql/schema.sql +++ b/schema/mysql/schema.sql @@ -1018,11 +1018,10 @@ CREATE TABLE kubernetes_instance ( CREATE TABLE config ( cluster_uuid binary(16) NOT NULL, `key` enum( + 'notifications.url', 'notifications.username', 'notifications.password', - 'notifications.url', 'notifications.kubernetes_web_url', - 'notifications.source_id', 'prometheus.url', 'prometheus.username', 'prometheus.password' From be15d3046cc6064f06714b5e5218818e512ac45b Mon Sep 17 00:00:00 2001 From: Johannes Rauh Date: Wed, 15 Jan 2025 10:44:24 +0100 Subject: [PATCH 13/13] Only validate if url is set --- pkg/metrics/config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/metrics/config.go b/pkg/metrics/config.go index a2e64e9..933894c 100644 --- a/pkg/metrics/config.go +++ b/pkg/metrics/config.go @@ -13,8 +13,9 @@ type PrometheusConfig struct { // Validate checks constraints in the supplied Prometheus configuration and returns an error if they are violated. func (c *PrometheusConfig) Validate() error { - if (c.Username == "") != (c.Password == "") { + if c.Url != "" && (c.Username == "") != (c.Password == "") { return errors.New("both username and password must be provided") } + return nil }