Merge pull request #24 from tuenti/release-0.2.0-rc.2

Cut Release 0.2.0 rc.2

.dockerignore

@@ -1,3 +1,4 @@
 vendor/
 build/
 Dockerfile
+.git

CHANGELOG.md

@@ -1,3 +1,14 @@
+## v0.2.0-rc.2 - 2019-01-29
+
+### Added 
+- New `secrets_manager_vault_max_token_ttl` metric, so a user could alert based on this and `secrets_manager_token_ttl`
+- New `secrets_manager_secret_last_sync_status` metric, that shows wether the secret succeded or not in last synchronization iteration
+
+### Fixed
+- Deprecates `secrets_manager_vault_token_expired` metric as it was quite confusing since it's not really possible for `secrets-manager` to know when the token it's expired, just when it's "close to expire".
+- Renames counter metrics to follow the Prometheus naming standard with the `_total` suffix instead of `_count`.
+- Simplifies prometheus token renewal metrics by merging `secrets_manager_vault_token_lookup_errors_count` and `secrets_manager_vault_token_renew_errors_count` into one single metric `secrets_manager_vault_token_renewal_errors_total` with one more dimension called `vault_operation` which will be one of `lookup-self, renew-self, is-renewable`.
+
 ## v0.2.0-rc.1 - 2019-01-21
 
 ### Added

Dockerfile

@@ -1,8 +1,14 @@
 # Stage 0
 # Build binary file
-FROM instrumentisto/glide:0.13.1-go1.10
+FROM golang:1.11.5-alpine as builder
+ENV GLIDE_VERSION v0.13.2
 
-RUN apk add --update make
+RUN wget "https://github.com/Masterminds/glide/releases/download/${GLIDE_VERSION}/glide-${GLIDE_VERSION}-linux-amd64.tar.gz" \
+    && tar xf glide-${GLIDE_VERSION}-linux-amd64.tar.gz \
+    && cp linux-amd64/glide $GOPATH/bin/ \
+    && chmod +x $GOPATH/bin/glide
+
+RUN apk add --update git make
 
 ARG PROJECT_SLUG=github.com/tuenti/secrets-manager
 COPY glide.yaml /go/src/$PROJECT_SLUG/glide.yaml
@@ -16,7 +22,8 @@ RUN make build-linux
 
 # Stage 1
 # Build actual docker image
-FROM alpine
+FROM alpine:3.8
 ARG PROJECT_SLUG=github.com/tuenti/secrets-manager
-COPY --from=0 /go/src/$PROJECT_SLUG/build/secrets-manager /secrets-manager
+LABEL maintainer="[email protected]"
+COPY --from=builder /go/src/$PROJECT_SLUG/build/secrets-manager /secrets-manager
 ENTRYPOINT ["/secrets-manager"]

Makefile

@@ -2,7 +2,7 @@
 # Binary names
 BUILD_FOLDER=build
 BINARY_NAME=secrets-manager
-SECRETS_MANAGER_VERSION=v0.2.0-rc.1
+SECRETS_MANAGER_VERSION=v0.2.0-rc.2
 DOCKER_REGISTRY ?= "registry.hub.docker.com"
 DOCKER_IMAGE_NAME=${BINARY_NAME}
 DOCKER_IMAGE=${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:${SECRETS_MANAGER_VERSION}

README.md

@@ -104,13 +104,14 @@ data:
 
 | Metric| Type| Description| Labels|
 | ------| ----|------------| ------|
-|`secrets_manager_vault_token_expired` | Gauge | Whether or not token TTL is under `vault.max-token-ttl`: 1 = expired; 0 = still valid | `"vault_address", "vault_engine", "vault_version", "vault_cluster_id", "vault_cluster_name"` |
+|`secrets_manager_vault_max_token_ttl` | Gauge | `secrets-manager` max Vault token TTL | `"vault_address", "vault_engine", "vault_version", "vault_cluster_id", "vault_cluster_name"` |
 |`secrets_manager_vault_token_ttl` | Gauge | Vault token TTL | `"vault_address", "vault_engine", "vault_version", "vault_cluster_id", "vault_cluster_name"` |
-|`secrets_manager_vault_token_lookup_errors_count`| Counter | Vault token lookup-self errors counter | `"vault_address", "vault_engine", "vault_version", "vault_cluster_id", "vault_cluster_name", "error"` |
-|`secrets_manager_vault_token_renew_errors_count`| Counter | Vault token renew-self errors counter | `"vault_address", "vault_engine", "vault_version", "vault_cluster_id", "vault_cluster_name", "error"` |
-|`secrets_manager_read_secret_errors_count`| Counter | Vault read operations counter | `"vault_address", "vault_engine", "vault_version", "vault_cluster_id", "vault_cluster_name", "path", "key", "error"` |
-| `secrets_manager_secret_sync_errors_count`| Counter |Secrets sync error counter|`"name", "namespace"`|
+|`secrets_manager_vault_token_renewal_errors_total`| Counter | Vault token renewal errors counter | `"vault_address", "vault_engine", "vault_version", "vault_cluster_id", "vault_cluster_name", "vault_operation", "error"` |
+|`secrets_manager_k8s_secret_read_errors_total`| Counter | Errors total count when reading a secret from Kubernetes | `"name", "namespace"` |
+|`secrets_manager_k8s_secret_update_errors_total`| Counter | Error total count when updating (and also creating) a secret in Kubernetes | `"name", "namespace"` |
+| `secrets_manager_secret_sync_errors_total`| Counter |Secrets synchronization total errors.|`"name", "namespace"`|
 |`secrets_manager_secret_last_updated`| Gauge |The last update timestamp as a Unix time (the number of seconds elapsed since January 1, 1970 UTC)|`"name", "namespace"`|
+|`secrets_manager_secret_last_sync_status`| Gauge |The result of the last sync of a secret. 1 = OK, 0 = Error|`"name", "namespace"`|
 
 ## Getting Started with Vault
 

backend/vault.go

@@ -72,6 +72,9 @@ func vaultClient(l *log.Logger, cfg Config) (*client, error) {
 		renewTTLIncrement:  cfg.VaultRenewTTLIncrement,
 		engine:             engine,
 	}
+
+	metrics.updateVaultMaxTokenTTLMetric(cfg.VaultMaxTokenTTL)
+
 	return &client, err
 }
 
@@ -80,7 +83,7 @@ func (c *client) getToken() (*api.Secret, error) {
 	lookup, err := auth.Token().LookupSelf()
 	if err != nil {
 		logger.Errorf("error checking token with lookup self api: %v", err)
-		metrics.updateVaultTokenLookupErrorsCountMetric(errors.UnknownErrorType)
+		metrics.updateVaultTokenRenewalErrorsTotalMetric(vaultLookupSelfOperationName, errors.UnknownErrorType)
 		return nil, err
 	}
 	return lookup, nil
@@ -97,31 +100,22 @@ func (c *client) getTokenTTL(token *api.Secret) (int64, error) {
 	return ttl, nil
 }
 
-func (c *client) shouldRenewToken(ttl int64) bool {
-	if ttl < c.maxTokenTTL {
-		metrics.updateVaultTokenExpiredMetric(vaultTokenExpired)
-		return true
-	}
-	metrics.updateVaultTokenExpiredMetric(vaultTokenNotExpired)
-	return false
-}
-
 func (c *client) renewToken(token *api.Secret) error {
 	isRenewable, err := token.TokenIsRenewable()
 	if err != nil {
 		logger.Errorf("could not check token renewability: %v", err)
-		metrics.updateVaultTokenRenewErrorsCountMetric(errors.UnknownErrorType)
+		metrics.updateVaultTokenRenewalErrorsTotalMetric(vaultIsRenewableOperationName, errors.UnknownErrorType)
 		return err
 	}
 	if !isRenewable {
-		metrics.updateVaultTokenRenewErrorsCountMetric(errors.VaultTokenNotRenewableErrorType)
+		metrics.updateVaultTokenRenewalErrorsTotalMetric(vaultIsRenewableOperationName, errors.VaultTokenNotRenewableErrorType)
 		err = &errors.VaultTokenNotRenewableError{ErrType: errors.VaultTokenNotRenewableErrorType}
 		return err
 	}
 	auth := c.vclient.Auth()
 	if _, err = auth.Token().RenewSelf(c.renewTTLIncrement); err != nil {
 		log.Errorf("failed to renew token: %v", err)
-		metrics.updateVaultTokenRenewErrorsCountMetric(errors.UnknownErrorType)
+		metrics.updateVaultTokenRenewalErrorsTotalMetric(vaultRenewSelfOperationName, errors.UnknownErrorType)
 		return err
 	}
 	return nil
@@ -137,7 +131,7 @@ func (c *client) renewalLoop() {
 	if err != nil {
 		logger.Errorf("failed to read token TTL: %v", err)
 		return
-	} else if c.shouldRenewToken(ttl) {
+	} else if ttl < c.maxTokenTTL {
 		logger.Warnf("token is really close to expire, current ttl: %d", ttl)
 		err := c.renewToken(token)
 		if err != nil {
@@ -174,7 +168,7 @@ func (c *client) ReadSecret(path string, key string) (string, error) {
 	logical := c.logical
 	secret, err := logical.Read(path)
 	if err != nil {
-		metrics.updateVaultSecretReadErrorsCountMetric(path, key, errors.UnknownErrorType)
+		metrics.updateVaultSecretReadErrorsTotalMetric(path, key, errors.UnknownErrorType)
 		return data, err
 	}
 
@@ -185,18 +179,18 @@ func (c *client) ReadSecret(path string, key string) (string, error) {
 			if secretData[key] != nil {
 				data = secretData[key].(string)
 			} else {
-				metrics.updateVaultSecretReadErrorsCountMetric(path, key, errors.BackendSecretNotFoundErrorType)
+				metrics.updateVaultSecretReadErrorsTotalMetric(path, key, errors.BackendSecretNotFoundErrorType)
 				err = &errors.BackendSecretNotFoundError{ErrType: errors.BackendSecretNotFoundErrorType, Path: path, Key: key}
 			}
 		} else {
 			for _, w := range warnings {
 				logger.Warningln(w)
 			}
-			metrics.updateVaultSecretReadErrorsCountMetric(path, key, errors.BackendSecretNotFoundErrorType)
+			metrics.updateVaultSecretReadErrorsTotalMetric(path, key, errors.BackendSecretNotFoundErrorType)
 			err = &errors.BackendSecretNotFoundError{ErrType: errors.BackendSecretNotFoundErrorType, Path: path, Key: key}
 		}
 	} else {
-		metrics.updateVaultSecretReadErrorsCountMetric(path, key, errors.BackendSecretNotFoundErrorType)
+		metrics.updateVaultSecretReadErrorsTotalMetric(path, key, errors.BackendSecretNotFoundErrorType)
 		err = &errors.BackendSecretNotFoundError{ErrType: errors.BackendSecretNotFoundErrorType, Path: path, Key: key}
 	}
 	return data, err

backend/vault_metrics.go

@@ -3,44 +3,39 @@ package backend
 import "github.com/prometheus/client_golang/prometheus"
 
 const (
-	vaultTokenExpired    = 1
-	vaultTokenNotExpired = 0
+	vaultLookupSelfOperationName  = "lookup-self"
+	vaultRenewSelfOperationName   = "renew-self"
+	vaultIsRenewableOperationName = "is-renewable"
 )
 
 var (
-	vaultLabelNames  = []string{"vault_address", "vault_engine", "vault_version", "vault_cluster_id", "vault_cluster_name"}
-	secretLabelNames = []string{"path", "key", "error"}
-	errorLabelNames  = []string{"error"}
+	vaultLabelNames      = []string{"vault_address", "vault_engine", "vault_version", "vault_cluster_id", "vault_cluster_name"}
+	secretLabelNames     = []string{"path", "key", "error"}
+	vaultErrorLabelNames = []string{"vault_operation", "error"}
 
 	// Prometeheus metrics: https://prometheus.io
-	tokenExpired = prometheus.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "secrets_manager",
-		Subsystem: "vault",
-		Name:      "token_expired",
-		Help:      "The state of the token: 1 = expired; 0 = still valid",
-	}, vaultLabelNames)
 	tokenTTL = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: "secrets_manager",
 		Subsystem: "vault",
 		Name:      "token_ttl",
 		Help:      "Vault token TTL",
 	}, vaultLabelNames)
-	tokenLookupErrorsCount = prometheus.NewCounterVec(prometheus.CounterOpts{
+	maxTokenTTL = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: "secrets_manager",
 		Subsystem: "vault",
-		Name:      "token_lookup_errors_count",
-		Help:      "Vault token lookup-self errors counter",
-	}, append(vaultLabelNames, errorLabelNames...))
-	tokenRenewErrorsCount = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name:      "max_token_ttl",
+		Help:      "secrets-manager max Vault token TTL",
+	}, vaultLabelNames)
+	tokenRenewalErrorsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "secrets_manager",
 		Subsystem: "vault",
-		Name:      "token_renew_errors_count",
-		Help:      "Vault token renew-self errors counter",
-	}, append(vaultLabelNames, errorLabelNames...))
-	secretReadErrorsCount = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name:      "token_renewal_errors_total",
+		Help:      "Vault token renewal errors counter",
+	}, append(vaultLabelNames, vaultErrorLabelNames...))
+	secretReadErrorsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "secrets_manager",
 		Subsystem: "vault",
-		Name:      "read_secret_errors_count",
+		Name:      "read_secret_errors_total",
 		Help:      "Vault read operations counter",
 	}, append(vaultLabelNames, secretLabelNames...))
 )
@@ -50,11 +45,10 @@ type vaultMetrics struct {
 }
 
 func init() {
-	prometheus.MustRegister(tokenExpired)
 	prometheus.MustRegister(tokenTTL)
-	prometheus.MustRegister(tokenLookupErrorsCount)
-	prometheus.MustRegister(tokenRenewErrorsCount)
-	prometheus.MustRegister(secretReadErrorsCount)
+	prometheus.MustRegister(maxTokenTTL)
+	prometheus.MustRegister(tokenRenewalErrorsTotal)
+	prometheus.MustRegister(secretReadErrorsTotal)
 }
 
 func newVaultMetrics(vaultAddr string, vaultVersion string, vaultEngine string, vaultClusterID string, vaultClusterName string) *vaultMetrics {
@@ -68,13 +62,8 @@ func newVaultMetrics(vaultAddr string, vaultVersion string, vaultEngine string,
 	return &vaultMetrics{vaultLabels: labels}
 }
 
-func (vm *vaultMetrics) updateVaultTokenExpiredMetric(value int) {
-	if value != vaultTokenExpired && value != vaultTokenNotExpired {
-		logger.Errorf("refusing to update secrets_manager_vault_token_expired metric with value %d. Allowed values are %d and %d", value, vaultTokenExpired, vaultTokenNotExpired)
-		return
-	}
-
-	tokenExpired.WithLabelValues(
+func (vm *vaultMetrics) updateVaultMaxTokenTTLMetric(value int64) {
+	maxTokenTTL.WithLabelValues(
 		vm.vaultLabels["vault_addr"],
 		vm.vaultLabels["vault_engine"],
 		vm.vaultLabels["vault_version"],
@@ -91,8 +80,8 @@ func (vm *vaultMetrics) updateVaultTokenTTLMetric(value int64) {
 		vm.vaultLabels["vault_cluster_name"]).Set(float64(value))
 }
 
-func (vm *vaultMetrics) updateVaultSecretReadErrorsCountMetric(path string, key string, errorType string) {
-	secretReadErrorsCount.WithLabelValues(
+func (vm *vaultMetrics) updateVaultSecretReadErrorsTotalMetric(path string, key string, errorType string) {
+	secretReadErrorsTotal.WithLabelValues(
 		vm.vaultLabels["vault_addr"],
 		vm.vaultLabels["vault_engine"],
 		vm.vaultLabels["vault_version"],
@@ -103,22 +92,13 @@ func (vm *vaultMetrics) updateVaultSecretReadErrorsCountMetric(path string, key
 		errorType).Inc()
 }
 
-func (vm *vaultMetrics) updateVaultTokenLookupErrorsCountMetric(errorType string) {
-	tokenLookupErrorsCount.WithLabelValues(
-		vm.vaultLabels["vault_addr"],
-		vm.vaultLabels["vault_engine"],
-		vm.vaultLabels["vault_version"],
-		vm.vaultLabels["vault_cluster_id"],
-		vm.vaultLabels["vault_cluster_name"],
-		errorType).Inc()
-}
-
-func (vm *vaultMetrics) updateVaultTokenRenewErrorsCountMetric(errorType string) {
-	tokenRenewErrorsCount.WithLabelValues(
+func (vm *vaultMetrics) updateVaultTokenRenewalErrorsTotalMetric(vaultOperation string, errorType string) {
+	tokenRenewalErrorsTotal.WithLabelValues(
 		vm.vaultLabels["vault_addr"],
 		vm.vaultLabels["vault_engine"],
 		vm.vaultLabels["vault_version"],
 		vm.vaultLabels["vault_cluster_id"],
 		vm.vaultLabels["vault_cluster_name"],
+		vaultOperation,
 		errorType).Inc()
 }

backend/vault_metrics_test.go

@@ -16,13 +16,13 @@ const (
 	fakeVaultClusterName = "vault-fake"
 )
 
-func TestUpdateTokenExpired(t *testing.T) {
+func TestUpdateMaxTokenTTL(t *testing.T) {
 	metrics := newVaultMetrics(fakeVaultAddress, fakeVaultVersion, fakeVaultEngine, fakeVaultClusterID, fakeVaultClusterName)
-	tokenExpired.Reset()
-	metrics.updateVaultTokenExpiredMetric(1)
-	metricTokenExpired, _ := tokenExpired.GetMetricWithLabelValues(fakeVaultAddress, fakeVaultEngine, fakeVaultVersion, fakeVaultClusterID, fakeVaultClusterName)
+	maxTokenTTL.Reset()
+	metrics.updateVaultMaxTokenTTLMetric(600)
+	metricMaxTokenTTL, _ := maxTokenTTL.GetMetricWithLabelValues(fakeVaultAddress, fakeVaultEngine, fakeVaultVersion, fakeVaultClusterID, fakeVaultClusterName)
 
-	assert.Equal(t, 1.0, testutil.ToFloat64(metricTokenExpired))
+	assert.Equal(t, 600.0, testutil.ToFloat64(metricMaxTokenTTL))
 }
 
 func TestUpdateTokenTTL(t *testing.T) {
@@ -34,44 +34,44 @@ func TestUpdateTokenTTL(t *testing.T) {
 	assert.Equal(t, 300.0, testutil.ToFloat64(metricTokenTTL))
 }
 
-func TestUpdateTokenLookupErrorsCount(t *testing.T) {
+func TestUpdateTokenLookupErrorsTotal(t *testing.T) {
 	metrics := newVaultMetrics(fakeVaultAddress, fakeVaultVersion, fakeVaultEngine, fakeVaultClusterID, fakeVaultClusterName)
-	tokenLookupErrorsCount.Reset()
-	metrics.updateVaultTokenLookupErrorsCountMetric(errors.UnknownErrorType)
-	metricTokenLookupErrorsCount, _ := tokenLookupErrorsCount.GetMetricWithLabelValues(fakeVaultAddress, fakeVaultEngine, fakeVaultVersion, fakeVaultClusterID, fakeVaultClusterName, errors.UnknownErrorType)
+	tokenRenewalErrorsTotal.Reset()
+	metrics.updateVaultTokenRenewalErrorsTotalMetric(vaultLookupSelfOperationName, errors.UnknownErrorType)
+	metricTokenRenewalErrorsTotal, _ := tokenRenewalErrorsTotal.GetMetricWithLabelValues(fakeVaultAddress, fakeVaultEngine, fakeVaultVersion, fakeVaultClusterID, fakeVaultClusterName, vaultLookupSelfOperationName, errors.UnknownErrorType)
 
-	assert.Equal(t, 1.0, testutil.ToFloat64(metricTokenLookupErrorsCount))
+	assert.Equal(t, 1.0, testutil.ToFloat64(metricTokenRenewalErrorsTotal))
 }
 
-func TestUpdateTokenRenewErrorsCount(t *testing.T) {
+func TestUpdateTokenRenewErrorsTotal(t *testing.T) {
 	metrics := newVaultMetrics(fakeVaultAddress, fakeVaultVersion, fakeVaultEngine, fakeVaultClusterID, fakeVaultClusterName)
-	tokenRenewErrorsCount.Reset()
-	metrics.updateVaultTokenRenewErrorsCountMetric(errors.UnknownErrorType)
-	metricTokenRenewErrorsCount, _ := tokenRenewErrorsCount.GetMetricWithLabelValues(fakeVaultAddress, fakeVaultEngine, fakeVaultVersion, fakeVaultClusterID, fakeVaultClusterName, errors.UnknownErrorType)
+	tokenRenewalErrorsTotal.Reset()
+	metrics.updateVaultTokenRenewalErrorsTotalMetric(vaultRenewSelfOperationName, errors.UnknownErrorType)
+	metricTokenRenewalErrorsTotal, _ := tokenRenewalErrorsTotal.GetMetricWithLabelValues(fakeVaultAddress, fakeVaultEngine, fakeVaultVersion, fakeVaultClusterID, fakeVaultClusterName, vaultRenewSelfOperationName, errors.UnknownErrorType)
 
-	assert.Equal(t, 1.0, testutil.ToFloat64(metricTokenRenewErrorsCount))
+	assert.Equal(t, 1.0, testutil.ToFloat64(metricTokenRenewalErrorsTotal))
 
-	tokenRenewErrorsCount.Reset()
-	metrics.updateVaultTokenRenewErrorsCountMetric(errors.VaultTokenNotRenewableErrorType)
-	metricTokenRenewErrorsCount, _ = tokenRenewErrorsCount.GetMetricWithLabelValues(fakeVaultAddress, fakeVaultEngine, fakeVaultVersion, fakeVaultClusterID, fakeVaultClusterName, errors.VaultTokenNotRenewableErrorType)
+	tokenRenewalErrorsTotal.Reset()
+	metrics.updateVaultTokenRenewalErrorsTotalMetric(vaultIsRenewableOperationName, errors.VaultTokenNotRenewableErrorType)
+	metricTokenRenewalErrorsTotal, _ = tokenRenewalErrorsTotal.GetMetricWithLabelValues(fakeVaultAddress, fakeVaultEngine, fakeVaultVersion, fakeVaultClusterID, fakeVaultClusterName, vaultIsRenewableOperationName, errors.VaultTokenNotRenewableErrorType)
 
-	assert.Equal(t, 1.0, testutil.ToFloat64(metricTokenRenewErrorsCount))
+	assert.Equal(t, 1.0, testutil.ToFloat64(metricTokenRenewalErrorsTotal))
 }
 
-func TestUpdateReadSecretErrorsCount(t *testing.T) {
+func TestUpdateReadSecretErrorsTotal(t *testing.T) {
 	path := "/path/to/secret"
 	key := "key"
 
 	metrics := newVaultMetrics(fakeVaultAddress, fakeVaultVersion, fakeVaultEngine, fakeVaultClusterID, fakeVaultClusterName)
-	secretReadErrorsCount.Reset()
-	metrics.updateVaultSecretReadErrorsCountMetric(path, key, errors.UnknownErrorType)
-	metricSecretReadErrorsCount, _ := secretReadErrorsCount.GetMetricWithLabelValues(fakeVaultAddress, fakeVaultEngine, fakeVaultVersion, fakeVaultClusterID, fakeVaultClusterName, path, key, errors.UnknownErrorType)
+	secretReadErrorsTotal.Reset()
+	metrics.updateVaultSecretReadErrorsTotalMetric(path, key, errors.UnknownErrorType)
+	metricSecretReadErrorsTotal, _ := secretReadErrorsTotal.GetMetricWithLabelValues(fakeVaultAddress, fakeVaultEngine, fakeVaultVersion, fakeVaultClusterID, fakeVaultClusterName, path, key, errors.UnknownErrorType)
 
-	assert.Equal(t, 1.0, testutil.ToFloat64(metricSecretReadErrorsCount))
+	assert.Equal(t, 1.0, testutil.ToFloat64(metricSecretReadErrorsTotal))
 
-	secretReadErrorsCount.Reset()
-	metrics.updateVaultSecretReadErrorsCountMetric(path, key, errors.BackendSecretNotFoundErrorType)
-	metricSecretReadErrorsCount, _ = secretReadErrorsCount.GetMetricWithLabelValues(fakeVaultAddress, fakeVaultEngine, fakeVaultVersion, fakeVaultClusterID, fakeVaultClusterName, path, key, errors.BackendSecretNotFoundErrorType)
+	secretReadErrorsTotal.Reset()
+	metrics.updateVaultSecretReadErrorsTotalMetric(path, key, errors.BackendSecretNotFoundErrorType)
+	metricSecretReadErrorsTotal, _ = secretReadErrorsTotal.GetMetricWithLabelValues(fakeVaultAddress, fakeVaultEngine, fakeVaultVersion, fakeVaultClusterID, fakeVaultClusterName, path, key, errors.BackendSecretNotFoundErrorType)
 
-	assert.Equal(t, 1.0, testutil.ToFloat64(metricSecretReadErrorsCount))
+	assert.Equal(t, 1.0, testutil.ToFloat64(metricSecretReadErrorsTotal))
 }