From 8043d0cc00001a8e31efdb71abd4a44db5a4df70 Mon Sep 17 00:00:00 2001 From: Angel Barrera Date: Sun, 18 Jun 2023 09:02:23 +0200 Subject: [PATCH] Add label support --- README.md | 51 +++++++++++++++++-- configs/sample.labels.yaml | 17 +++++++ .../prometheus-multi-tenant-proxy/basic.go | 12 ++++- .../app/prometheus-multi-tenant-proxy/jwt.go | 24 ++++----- .../prometheus-multi-tenant-proxy/jwt_test.go | 29 +++++++---- .../prometheus-multi-tenant-proxy/reverse.go | 42 ++++++++++----- internal/pkg/config_test.go | 42 +++++++++++++++ 7 files changed, 173 insertions(+), 44 deletions(-) create mode 100644 configs/sample.labels.yaml diff --git a/README.md b/README.md index be48a7b..2ed6d82 100644 --- a/README.md +++ b/README.md @@ -57,10 +57,11 @@ type Authn struct { // User Identifies a user including the tenant type User struct { - Username string `yaml:"username"` - Password string `yaml:"password"` - Namespace string `yaml:"namespace"` - Namespaces []string `yaml:"namespaces"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Namespace string `yaml:"namespace"` + Namespaces []string `yaml:"namespaces"` + Labels map[string]string `yaml:"labels"` } ``` @@ -104,6 +105,31 @@ users: A tenant can contain multiple users. But a user is tied to a single tenant. +Tenant definition usually contains a set of labels. Starting from v1.7.0 it's possible to add these labels to a new `labels` +section to the user definition to inject these labels on queries for that user. + +Example available at [configs/sample.labels.yaml](configs/sample.labels.yaml) file: + +```yaml +users: + - username: Happy + password: Prometheus + labels: + app: happy + team: america + - username: Sad + password: Prometheus + labels: + namespace: kube-system + - username: bored + password: Prometheus + namespaces: + - default + - kube-system + labels: + dep: system +``` + #### Configure the proxy for JWT authentication Under the hood, the proxy uses [keyfunc](https://github.com/MicahParks/keyfunc) to load @@ -126,12 +152,21 @@ The **token** is extracted from one of two locations with the given precedence: For the token to be valid, it must: * contain a `kid` (key ID) in the header that matches the kid of a known key in the JWKS, -* contain a claim in the payload called `namespaces`, with one or more values. For example: +* contain a claim in the payload called `namespaces`, with zero or more values. For example: ```json { "namespaces": ["foo", "bar"] } ``` +* contain a claim in the payload called `labels`, with zero or more values. For example: + ```json + { + "labels": { + "app": "happy", + "team": "america" + } + } + ``` * have been signed with the key in the JWKS matching the `kid` found in the JWT header. To test the proxy using JWT tokens, you can use the `.jwks_example.json` file above to run @@ -146,6 +181,12 @@ You can now use curl, for example: curl -H "Authorization: Bearer $TOKEN" http://localhost:9092/api/v1/query\?query\=net_conntrack_dialer_conn_attempted_total ``` +#### Namespaces or labels + +The proxy can be configured to use either namespaces and/or labels to query Prometheus. +At least one must be configured, otherwise the proxy will not proxy the query to Prometheus. +*(It could lead to a security issue if the proxy is not configured to use namespaces or labels)* + ## Build it If you want to build it from this repository, follow the instructions below: diff --git a/configs/sample.labels.yaml b/configs/sample.labels.yaml new file mode 100644 index 0000000..1fffd31 --- /dev/null +++ b/configs/sample.labels.yaml @@ -0,0 +1,17 @@ +users: + - username: Happy + password: Prometheus + labels: + app: happy + team: america + - username: Sad + password: Prometheus + labels: + namespace: kube-system + - username: bored + password: Prometheus + namespaces: + - default + - kube-system + labels: + dep: system diff --git a/internal/app/prometheus-multi-tenant-proxy/basic.go b/internal/app/prometheus-multi-tenant-proxy/basic.go index f57a831..35da747 100644 --- a/internal/app/prometheus-multi-tenant-proxy/basic.go +++ b/internal/app/prometheus-multi-tenant-proxy/basic.go @@ -74,7 +74,17 @@ func (auth *BasicAuth) isAuthorized(user, pass string) (bool, []string, map[stri authConfig := auth.getConfig() for _, v := range authConfig.Users { if subtle.ConstantTimeCompare([]byte(user), []byte(v.Username)) == 1 && subtle.ConstantTimeCompare([]byte(pass), []byte(v.Password)) == 1 { - return true, append(v.Namespaces, v.Namespace), nil + // User is authorized, return the namespaces + namespaces := make([]string, 0) + // If the user has a namespace, add it to the list + if v.Namespace != "" { + namespaces = append(namespaces, v.Namespace) + } + // If the user has namespaces, add them to the list + if v.Namespaces != nil { + namespaces = append(namespaces, v.Namespaces...) + } + return true, namespaces, v.Labels } } return false, nil, nil diff --git a/internal/app/prometheus-multi-tenant-proxy/jwt.go b/internal/app/prometheus-multi-tenant-proxy/jwt.go index b7a6eff..8fb7930 100644 --- a/internal/app/prometheus-multi-tenant-proxy/jwt.go +++ b/internal/app/prometheus-multi-tenant-proxy/jwt.go @@ -4,9 +4,9 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io/ioutil" "log" "net/http" + "os" "strings" "sync" @@ -17,7 +17,8 @@ import ( // NamespaceClaim expected structure of the JWT token payload type NamespaceClaim struct { // Namespaces contains the list of namespaces a user has access to - Namespaces []string `json:"namespaces"` + Namespaces []string `json:"namespaces"` + Labels map[string]string `json:"labels"` jwt.RegisteredClaims } @@ -103,7 +104,7 @@ func (auth *JwtAuth) loadFromURL(url *string) bool { } func (auth *JwtAuth) loadFromFile(location *string) bool { - content, err := ioutil.ReadFile(*location) + content, err := os.ReadFile(*location) if err != nil { log.Printf("Failed to read JWKS file: %v", err) return false @@ -152,20 +153,13 @@ func (auth *JwtAuth) isAuthorized(tokenString string) (bool, []string, map[strin } claims := token.Claims.(*NamespaceClaim) - if len(claims.Namespaces) == 0 { - log.Printf("token claim is invalid: namespaces is missing or empty") - return false, nil, nil + if claims.Namespaces == nil { + claims.Namespaces = []string{} } - return true, claims.Namespaces, nil -} - -func isValidSigningMethod(signingMethod string) bool { - for _, alg := range jwt.GetAlgorithms() { - if signingMethod == alg { - return true - } + if claims.Labels == nil { + claims.Labels = make(map[string]string) } - return false + return true, claims.Namespaces, claims.Labels } func extractTokens(headers *http.Header) string { diff --git a/internal/app/prometheus-multi-tenant-proxy/jwt_test.go b/internal/app/prometheus-multi-tenant-proxy/jwt_test.go index aa16cfc..00c9407 100644 --- a/internal/app/prometheus-multi-tenant-proxy/jwt_test.go +++ b/internal/app/prometheus-multi-tenant-proxy/jwt_test.go @@ -2,7 +2,6 @@ package proxy import ( _ "embed" - "io/ioutil" "net/http" "net/http/httptest" "os" @@ -30,8 +29,15 @@ var ( const ( // kid = hmac-key, payload = {"namespaces": ["prometheus"]} validHmacToken = "eyJhbGciOiJIUzI1NiIsImtpZCI6ImhtYWMta2V5In0.eyJuYW1lc3BhY2VzIjpbInByb21ldGhldXMiXX0.mGc9neZ2-C6fOXwI_h5Qknj-lH1apcFKVUo0-WlDPss" + // rsa jwt geneator online: https://www.scottbrady91.com/tools/jwt // kid = "rs256-key", payload = {"namespaces": ["prometheus", "app-1"]}} validRsaToken = "eyJhbGciOiJSUzI1NiIsImtpZCI6InJzMjU2LWtleSIsInR5cCI6IkpXVCJ9.eyJuYW1lc3BhY2VzIjpbInByb21ldGhldXMiLCJhcHAtMSJdfQ.n_hy5yqjFkpD00VNGCLkRyeOBdcjeu9Yp1TVzV5jSKaX32Idrl2jv1mHCX5JJfM-tyLXxCQJcze9q7IXpN0_x-E7iE_uAvDT7BiMWSwy7lWW2eRuffggv2EG8HP3_kGgsH-RcP4B5VbaKeB9N1RNrHwvxoiYKhcFQCTJzsf010s10nUYmfL0jQ8hW--yTX2kly8zXxBoJXu6rluNMXWL7o8Tx9ONHLLlz-trP7s9xFN_GQtbZ3lKZ5n8XESccctXWAdIqtYtlTA4KCr0krIX7cRbLdni5QOPBTwQxdOBujdDaXZqo8K8PJfaZ93oyJUdYe7rnX0Lz_dT1EJLWYvm-A" + // kid = "rs256-key", payload = {"namespaces": []}} + validEmptyNamespacesRSAToken = "eyJhbGciOiJSUzI1NiIsImtpZCI6InJzMjU2LWtleSIsInR5cCI6IkpXVCJ9.eyJuYW1lc3BhY2VzIjpbXX0.LWPlxmZPDaKA3Z-IbRAoBYuymCx3cdZvXHzlSfVIhj4TjoQZ8Rom5IWtJpoEiq-_DkQHFgFRnLTsFE8CcaYM_eLWRZPK7c_rDwzfJDxDVhIL3k6krL5gq_4Y6nOGnjktJkIJvJstl9FDc7gyx0EBvUX-cgQzh-my9whMXBrZ0oybVyiBGlAZbVOiW-BObm3U0hYF4Xt6HOTm4khAEsZPnS4rglQpQki_q4w67OaMcTwfO_hr6KJtwzavLLCWJhijWdON93ueubn4Z294TM5SWQFzPM-knFDaBfzq5k94NQviBoT7ekb9RsGLrjKsrVOdOVMM8b4BEFXMtZpVENLgQg" + // kid = "rs256-key", payload = {"labels": {"app":"ecom","team":"europe"}}} + validTwoLabelsRSAToken = "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6InJzMjU2LWtleSJ9.eyJsYWJlbHMiOnsiYXBwIjoiZWNvbSIsInRlYW0iOiJldXJvcGUifX0.Xb9-WPdP-yL7afsYShGQt1p3YVhNcufY-6dxtCVnbhgKLotqgy81tS-5RxF7KdSlSkfuwNyZCuE_qnKO_seOxczHOkARWnvZ5jlfPoPI8adKiVykeDR6q6fj3fO5Mp7BDNVXBwb9_wQ08Y3JwONdoNmvdnUz6aspD7IVIL41t64kst-GTxvvkdA-1Xfh9LB0zmyaCEgYiaByNJevtqnwFociTzRbWR2yXcEkhzbqKSGG6ia55It5CeN3GB9sjAWOEd57fSgDJwr0D80zxFoXtLeX64gcCjNsxJsh5ZrQ8U34fdo-73mPDJPCOBkowiamPDWOkBQ54U5lesbE5R3KPA" + // kid = "rs256-key", payload = {"labels":{"app":"ecom","team":"europe"},"namespaces":["kube-system","monitoring"]} + validTwoLabelsTwoNamespacesRSAToken = "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6InJzMjU2LWtleSJ9.eyJsYWJlbHMiOnsiYXBwIjoiZWNvbSIsInRlYW0iOiJldXJvcGUifSwibmFtZXNwYWNlcyI6WyJrdWJlLXN5c3RlbSIsIm1vbml0b3JpbmciXX0.Zk6hE9OBUIH5ctMzSeq2p40dJFiwZS_TghePWlTB1_-XHOzZRGvbT-sXoZnIy1__lHJZ4h8t0-P0_zwQPpZ2aB2A0Ar3wogEiIdktoRtqQcMvSjjIjwNm8e9uaE1QBpeqNtxg5i3hDMJLVfsoXta0PJ9YW4hbuhnpaThhji9M7duOXv9eeW4nJHSFr3YVCn75qR35O8z3Pwjo_06OhSpK5sy1PbqQLNvzkWdKYiqAjezWnnh6kO37hQfDJWUaKxkhE4TmOMJRk_mRKrUpHZ1mQ6rZ4YXyo0pBBNqJ5uJYA45bT2FJpNqJ9rXHf2qjDBwcS6SEw8pDe-iRdIC0xr1Ig" ) func (auth *JwtAuth) assertHmac(t *testing.T, expectAuthorized bool) { @@ -94,7 +100,7 @@ func TestJWT_LoadFromFile(t *testing.T) { defer os.Remove(file.Name()) // Load only the HMAC key - ioutil.WriteFile(file.Name(), []byte(jwksHMAC), 0644) + os.WriteFile(file.Name(), []byte(jwksHMAC), 0644) auth := NewJwtAuth(file.Name()) if !auth.isFile { t.Fatal("auth.isFile should be false") @@ -103,7 +109,7 @@ func TestJWT_LoadFromFile(t *testing.T) { auth.assertRSA(t, false) // Reload and trigger and error, it should still work - ioutil.WriteFile(file.Name(), []byte(""), 0644) + os.WriteFile(file.Name(), []byte(""), 0644) if auth.Load() { t.Error("The load should have failed") } @@ -111,7 +117,7 @@ func TestJWT_LoadFromFile(t *testing.T) { auth.assertRSA(t, false) // Reload with all keys this time - ioutil.WriteFile(file.Name(), []byte(jwksJSON), 0644) + os.WriteFile(file.Name(), []byte(jwksJSON), 0644) if !auth.Load() { t.Error("The load should have succeeded") } @@ -125,20 +131,27 @@ func TestJWT_IsAuthorized(t *testing.T) { desc string token string ns []string + l map[string]string }{ - {"hmac", validHmacToken, []string{"prometheus"}}, - {"rsa", validRsaToken, []string{"prometheus", "app-1"}}, + {"hmac", validHmacToken, []string{"prometheus"}, map[string]string{}}, + {"rsa", validRsaToken, []string{"prometheus", "app-1"}, map[string]string{}}, + {"empty-namespace-rsa", validEmptyNamespacesRSAToken, []string{}, map[string]string{}}, + {"two-labels-rsa", validTwoLabelsRSAToken, []string{}, map[string]string{"app": "ecom", "team": "europe"}}, + {"two-labels-two-namespaces-rsa", validTwoLabelsTwoNamespacesRSAToken, []string{"kube-system", "monitoring"}, map[string]string{"app": "ecom", "team": "europe"}}, } for _, tc := range validTestCases { t.Run(tc.desc, func(t *testing.T) { - authorized, namespaces, _ := auth.isAuthorized(tc.token) + authorized, namespaces, labels := auth.isAuthorized(tc.token) if !authorized { t.Fatal("Should be authorized") } if !reflect.DeepEqual(namespaces, tc.ns) { t.Fatalf("Got unexpected namespace: %v", namespaces) } + if !reflect.DeepEqual(labels, tc.l) { + t.Fatalf("Got unexpected labels: %v", labels) + } }) } @@ -149,9 +162,7 @@ func TestJWT_IsAuthorized(t *testing.T) { {"empty", ""}, // Empty JWT. {"wrong key", "eyJhbGciOiJIUzI1NiIsImtpZCI6ImhtYWMta2V5In0.eyJuYW1lc3BhY2UiOiJwcm9tZXRoZXVzIn0.dY7Pwl4LLrBFkrK2krsYfj0PZdJSxHPSEtXGFozdhv0"}, {"wrong kid", "eyJhbGciOiJIUzI1NiIsImtpZCI6InVua25vd24ifQ.eyJuYW1lc3BhY2UiOiJwcm9tZXRoZXVzIn0.IijHPJ7xExe_CTXJ0A1M9qwOCelnSuMkD8AV4JzvD8M"}, - {"claim missing", "eyJhbGciOiJIUzI1NiIsImtpZCI6ImhtYWMta2V5In0.eyJmb28iOiJiYXIifQ.X_-BfA_HEqEDDYpZBN06538rMlJq80ODU7DsBFA9p_E"}, {"claim wrong type", "eyJhbGciOiJIUzI1NiIsImtpZCI6ImhtYWMta2V5In0.eyJuYW1lc3BhY2VzIjp0cnVlfQ.oZNkqDopM6DVMADg-utHeAolMhfWmlUlxL88a9yOB0M"}, - {"claim empty", "eyJhbGciOiJIUzI1NiIsImtpZCI6ImhtYWMta2V5In0.eyJuYW1lc3BhY2VzIjpbXX0.bhrLp8q57llzwITZ2dR4d6UW4Hfa9Q5KyO3SSFLhPc8"}, } for _, tc := range invalidTestCases { diff --git a/internal/app/prometheus-multi-tenant-proxy/reverse.go b/internal/app/prometheus-multi-tenant-proxy/reverse.go index 84090ea..f83d216 100644 --- a/internal/app/prometheus-multi-tenant-proxy/reverse.go +++ b/internal/app/prometheus-multi-tenant-proxy/reverse.go @@ -45,30 +45,36 @@ func (r *ReversePrometheusRoundTripper) Director(req *http.Request) { func (r *ReversePrometheusRoundTripper) modifyRequest(req *http.Request, prometheusFormParameter string) error { namespaces := req.Context().Value(Namespaces).([]string) - if len(namespaces) == 0 { - return nil + l := req.Context().Value(Labels).(map[string]string) + + // Convert the labels map into a slice of label matchers. + var labelMatchers []*labels.Matcher + + for k, v := range l { + labelMatchers = append(labelMatchers, &labels.Matcher{ + Name: k, + Type: labels.MatchEqual, + Value: v, + }) } - var matcher labels.Matcher if len(namespaces) == 1 { // If there is only one namespace, we can use the more efficient MatchEqual matcher. - matcher = labels.Matcher{ + labelMatchers = append(labelMatchers, &labels.Matcher{ Name: "namespace", Type: labels.MatchEqual, Value: namespaces[0], - } - } else { + }) + } else if len(namespaces) > 1 { // If there are multiple namespaces, we need to use the MatchRegexp matcher. - matcher = labels.Matcher{ + labelMatchers = append(labelMatchers, &labels.Matcher{ Name: "namespace", Type: labels.MatchRegexp, Value: strings.Join(namespaces, "|"), - } + }) } - e := injector.NewEnforcer(false, []*labels.Matcher{ - &matcher, - }...) + e := injector.NewEnforcer(false, labelMatchers...) if err := req.ParseForm(); err != nil { return err @@ -83,10 +89,18 @@ func (r *ReversePrometheusRoundTripper) modifyRequest(req *http.Request, prometh if err != nil { return err } - if err := e.EnforceNode(expr); err != nil { - return err + log.Printf("[QUERY]\t%s ORIGINAL: %s\n", req.RemoteAddr, expr) + if len(namespaces) == 0 && len(l) == 0 { + log.Printf("[ERROR]\t%s\n", "no namespaces or labels found in request context") + // This is a hack to prevent the query from being executed. + value = "" + } else { + if err := e.EnforceNode(expr); err != nil { + return err + } + value = expr.String() } - value = expr.String() + log.Printf("[QUERY]\t%s MODIFIED: %s\n", req.RemoteAddr, value) } form.Set(key, value) } diff --git a/internal/pkg/config_test.go b/internal/pkg/config_test.go index 430db63..c06b671 100644 --- a/internal/pkg/config_test.go +++ b/internal/pkg/config_test.go @@ -11,6 +11,8 @@ func TestParseConfig(t *testing.T) { configSampleLocation := "../../configs/sample.yaml" configMultipleUserLocation := "../../configs/multiple.user.yaml" configMultipleNamespacesLocation := "../../configs/multiple.namespaces.yaml" + configSampleLabelsLocation := "../../configs/sample.labels.yaml" + expectedSampleAuth := Authn{ []User{ { @@ -28,6 +30,39 @@ func TestParseConfig(t *testing.T) { }, }, } + expectedSampleLabelsAuth := Authn{ + []User{ + { + Username: "Happy", + Password: "Prometheus", + Namespace: "", + Labels: map[string]string{ + "app": "happy", + "team": "america", + }, + Namespaces: []string{}, + }, { + Username: "Sad", + Password: "Prometheus", + Namespace: "", + Labels: map[string]string{ + "namespace": "kube-system", + }, + Namespaces: []string{}, + }, { + Username: "bored", + Password: "Prometheus", + Namespace: "", + Labels: map[string]string{ + "dep": "system", + }, + Namespaces: []string{ + "default", + "kube-system", + }, + }, + }, + } expectedMultipleUserAuth := Authn{ []User{ { @@ -102,6 +137,13 @@ func TestParseConfig(t *testing.T) { }, &expectedSampleAuth, false, + }, { + "Labels", + args{ + &configSampleLabelsLocation, + }, + &expectedSampleLabelsAuth, + false, }, { "Multiples users", args{