From 3a11e147f0909ad41f5afcd55b0da7471d6ecdff Mon Sep 17 00:00:00 2001 From: Dennis Ploeger Date: Wed, 30 Oct 2024 13:24:23 +0100 Subject: [PATCH] feat: First version --- .github/workflows/build_image.yaml | 91 ++++++++++++++++++ .github/workflows/release_chart.yaml | 32 +++++++ Dockerfile | 14 +++ charts/cloudian-exporter/.helmignore | 23 +++++ charts/cloudian-exporter/Chart.yaml | 6 ++ charts/cloudian-exporter/templates/NOTES.txt | 22 +++++ .../cloudian-exporter/templates/_helpers.tpl | 62 +++++++++++++ .../templates/deployment.yaml | 61 ++++++++++++ charts/cloudian-exporter/templates/hpa.yaml | 32 +++++++ .../cloudian-exporter/templates/ingress.yaml | 61 ++++++++++++ .../cloudian-exporter/templates/service.yaml | 15 +++ .../templates/serviceaccount.yaml | 12 +++ .../templates/servicemonitor.yaml | 14 +++ .../templates/tests/test-connection.yaml | 15 +++ charts/cloudian-exporter/values.yaml | 93 +++++++++++++++++++ cmd/cloudian-exporter.go | 69 ++++++++++++++ go.mod | 20 ++++ go.sum | 38 ++++++++ internal/api.go | 91 ++++++++++++++++++ internal/exporter.go | 69 ++++++++++++++ 20 files changed, 840 insertions(+) create mode 100644 .github/workflows/build_image.yaml create mode 100644 .github/workflows/release_chart.yaml create mode 100644 Dockerfile create mode 100644 charts/cloudian-exporter/.helmignore create mode 100644 charts/cloudian-exporter/Chart.yaml create mode 100644 charts/cloudian-exporter/templates/NOTES.txt create mode 100644 charts/cloudian-exporter/templates/_helpers.tpl create mode 100644 charts/cloudian-exporter/templates/deployment.yaml create mode 100644 charts/cloudian-exporter/templates/hpa.yaml create mode 100644 charts/cloudian-exporter/templates/ingress.yaml create mode 100644 charts/cloudian-exporter/templates/service.yaml create mode 100644 charts/cloudian-exporter/templates/serviceaccount.yaml create mode 100644 charts/cloudian-exporter/templates/servicemonitor.yaml create mode 100644 charts/cloudian-exporter/templates/tests/test-connection.yaml create mode 100644 charts/cloudian-exporter/values.yaml create mode 100644 cmd/cloudian-exporter.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/api.go create mode 100644 internal/exporter.go diff --git a/.github/workflows/build_image.yaml b/.github/workflows/build_image.yaml new file mode 100644 index 0000000..a4a56c0 --- /dev/null +++ b/.github/workflows/build_image.yaml @@ -0,0 +1,91 @@ +name: "Build image" + +on: + push: + branches: [ "main" ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + pull_request: + branches: [ "main" ] + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0 + with: + cosign-release: 'v2.2.4' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/.github/workflows/release_chart.yaml b/.github/workflows/release_chart.yaml new file mode 100644 index 0000000..d05f69d --- /dev/null +++ b/.github/workflows/release_chart.yaml @@ -0,0 +1,32 @@ +name: release-chart + +on: + push: + branches: + - main + paths: + - "charts/**" + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v1 + with: + version: v3.8.1 + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.4.0 + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a614fdb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM golang:1.23-alpine AS builder + +WORKDIR /app +COPY . . +RUN go build -o cloudian-exporter cmd/cloudian-exporter.go + +FROM alpine + +COPY --from=builder /app/cloudian-exporter / +RUN adduser -D cloudian-exporter && chmod +x /cloudian-exporter + +USER cloudian-exporter +EXPOSE 8080 +ENTRYPOINT ["/cloudian-exporter"] \ No newline at end of file diff --git a/charts/cloudian-exporter/.helmignore b/charts/cloudian-exporter/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/cloudian-exporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/cloudian-exporter/Chart.yaml b/charts/cloudian-exporter/Chart.yaml new file mode 100644 index 0000000..e2697d1 --- /dev/null +++ b/charts/cloudian-exporter/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: cloudian-exporter +description: A Helm chart for cloudian-exporter +type: application +version: 0.1.0 +appVersion: "latest" diff --git a/charts/cloudian-exporter/templates/NOTES.txt b/charts/cloudian-exporter/templates/NOTES.txt new file mode 100644 index 0000000..24e17b4 --- /dev/null +++ b/charts/cloudian-exporter/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "cloudian-exporter.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "cloudian-exporter.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "cloudian-exporter.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "cloudian-exporter.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/charts/cloudian-exporter/templates/_helpers.tpl b/charts/cloudian-exporter/templates/_helpers.tpl new file mode 100644 index 0000000..6ec86b3 --- /dev/null +++ b/charts/cloudian-exporter/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cloudian-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cloudian-exporter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cloudian-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cloudian-exporter.labels" -}} +helm.sh/chart: {{ include "cloudian-exporter.chart" . }} +{{ include "cloudian-exporter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cloudian-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cloudian-exporter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "cloudian-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "cloudian-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/cloudian-exporter/templates/deployment.yaml b/charts/cloudian-exporter/templates/deployment.yaml new file mode 100644 index 0000000..76a0097 --- /dev/null +++ b/charts/cloudian-exporter/templates/deployment.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "cloudian-exporter.fullname" . }} + labels: + {{- include "cloudian-exporter.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "cloudian-exporter.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "cloudian-exporter.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "cloudian-exporter.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/cloudian-exporter/templates/hpa.yaml b/charts/cloudian-exporter/templates/hpa.yaml new file mode 100644 index 0000000..27ade3a --- /dev/null +++ b/charts/cloudian-exporter/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "cloudian-exporter.fullname" . }} + labels: + {{- include "cloudian-exporter.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "cloudian-exporter.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/cloudian-exporter/templates/ingress.yaml b/charts/cloudian-exporter/templates/ingress.yaml new file mode 100644 index 0000000..17cfeb2 --- /dev/null +++ b/charts/cloudian-exporter/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "cloudian-exporter.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "cloudian-exporter.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/cloudian-exporter/templates/service.yaml b/charts/cloudian-exporter/templates/service.yaml new file mode 100644 index 0000000..616fdca --- /dev/null +++ b/charts/cloudian-exporter/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "cloudian-exporter.fullname" . }} + labels: + {{- include "cloudian-exporter.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "cloudian-exporter.selectorLabels" . | nindent 4 }} diff --git a/charts/cloudian-exporter/templates/serviceaccount.yaml b/charts/cloudian-exporter/templates/serviceaccount.yaml new file mode 100644 index 0000000..970dafe --- /dev/null +++ b/charts/cloudian-exporter/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "cloudian-exporter.serviceAccountName" . }} + labels: + {{- include "cloudian-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/cloudian-exporter/templates/servicemonitor.yaml b/charts/cloudian-exporter/templates/servicemonitor.yaml new file mode 100644 index 0000000..ec1b621 --- /dev/null +++ b/charts/cloudian-exporter/templates/servicemonitor.yaml @@ -0,0 +1,14 @@ +{{- if .Values.serviceMonitor.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "cloudian-exporter.fullname" . }} + labels: + {{- include "cloudian-exporter.labels" . | nindent 4 }} +spec: + endpoints: + - port: http + selector: + matchLabels: + {{- include "cloudian-exporter.labels" . | nindent 14 }} +{{-end-}} \ No newline at end of file diff --git a/charts/cloudian-exporter/templates/tests/test-connection.yaml b/charts/cloudian-exporter/templates/tests/test-connection.yaml new file mode 100644 index 0000000..2dd09e4 --- /dev/null +++ b/charts/cloudian-exporter/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "cloudian-exporter.fullname" . }}-test-connection" + labels: + {{- include "cloudian-exporter.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "cloudian-exporter.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/cloudian-exporter/values.yaml b/charts/cloudian-exporter/values.yaml new file mode 100644 index 0000000..eb2d41a --- /dev/null +++ b/charts/cloudian-exporter/values.yaml @@ -0,0 +1,93 @@ +# Default values for cloudian-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: ghcr.io/dodevops/cloudian-exporter/cloudian-exporter + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 8080 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +configuration: + cloudian: + url: "" + authSecret: "" + exporter: + loglevel: "" + refresh: 5 + +serviceMonitor: + enabled: false diff --git a/cmd/cloudian-exporter.go b/cmd/cloudian-exporter.go new file mode 100644 index 0000000..b4d9c33 --- /dev/null +++ b/cmd/cloudian-exporter.go @@ -0,0 +1,69 @@ +package main + +import ( + "cloudian-exporter/internal" + "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "log" + "net/http" + "os" + "time" +) +import "github.com/sirupsen/logrus" + +func main() { + logLevel := os.Getenv("EXPORTER_LOGLEVEL") + if logLevel == "" { + logLevel = "info" + } + if l, err := logrus.ParseLevel(logLevel); err != nil { + log.Fatalf("Can not parse loglevel %s: %v", logLevel, err) + } else { + logrus.SetLevel(l) + } + + logrus.SetFormatter(&logrus.JSONFormatter{}) + + baseURL := os.Getenv("CLOUDIAN_URL") + username := os.Getenv("CLOUDIAN_USERNAME") + password := os.Getenv("CLOUDIAN_PASSWORD") + + if baseURL == "" { + logrus.Fatal("Missing base url to Cloudian. Please set CLOUDIAN_URL") + } + if username == "" { + logrus.Fatal("Missing username to Cloudian. Please set CLOUDIAN_USERNAME") + } + if password == "" { + logrus.Fatal("Missing username to Cloudian. Please set CLOUDIAN_PASSWORD") + } + api := internal.NewCloudianAPI(baseURL, username, password) + + refresh := 5 * time.Minute + if r, ok := os.LookupEnv("EXPORTER_REFRESH"); ok { + if pr, err := time.ParseDuration(fmt.Sprintf("%sm", r)); err != nil { + logrus.Fatal("Can not parse refresh value from EXPORTER_REFRESH (%s): %v", r, err) + } else { + refresh = pr + } + } + + reg := prometheus.NewRegistry() + exit := make(chan bool) + + exporter := internal.NewCloudianExporter(refresh, api, exit, reg) + logrus.Info("Starting cloudian exporter") + exporter.Run() + + listen := os.Getenv("EXPORTER_LISTEN") + if listen == "" { + listen = ":8080" + } + + logrus.Info("Starting metrics server") + http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg})) + if err := http.ListenAndServe(listen, nil); err != nil { + logrus.Fatal("Can not start metrics server: %v", err) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..26699a6 --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module cloudian-exporter + +go 1.23.2 + +require ( + github.com/go-resty/resty/v2 v2.15.3 + github.com/prometheus/client_golang v1.19.1 + github.com/sirupsen/logrus v1.9.3 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/sys v0.22.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..844d3a0 --- /dev/null +++ b/go.sum @@ -0,0 +1,38 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-resty/resty/v2 v2.15.3 h1:bqff+hcqAflpiF591hhJzNdkRsFhlB96CYfBwSFvql8= +github.com/go-resty/resty/v2 v2.15.3/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/api.go b/internal/api.go new file mode 100644 index 0000000..e0d754f --- /dev/null +++ b/internal/api.go @@ -0,0 +1,91 @@ +package internal + +import ( + "fmt" + "github.com/go-resty/resty/v2" + "github.com/sirupsen/logrus" +) + +type CloudianAPI struct { + client *resty.Client +} + +func NewCloudianAPI(url string, username string, password string) CloudianAPI { + return CloudianAPI{ + client: resty.New(). + SetBaseURL(url). + SetBasicAuth(username, password). + SetLogger(logrus.StandardLogger()), + } +} + +type UserBuckets struct { + UserID string + Bucket string +} + +func (c CloudianAPI) GetBuckets(groupId string) ([]UserBuckets, error) { + var bucketResponse []struct { + UserID string `json:"userId"` + Buckets []struct { + BucketName string `json:"bucketName"` + } `json:"buckets"` + } + if r, err := c.client.R(). + SetResult(&bucketResponse). + SetQueryParam("groupId", groupId). + Get("/system/bucketlist"); err != nil { + return []UserBuckets{}, err + } else { + if r.IsError() { + return []UserBuckets{}, fmt.Errorf("error fetching buckets for group %s: (%s) %s", groupId, r.Status(), r.Body()) + } + var result []UserBuckets + for _, g := range bucketResponse { + for _, bucket := range g.Buckets { + result = append(result, UserBuckets{ + Bucket: bucket.BucketName, + UserID: g.UserID, + }) + } + } + return result, nil + } +} + +func (c CloudianAPI) GetGroups() ([]string, error) { + var groupsResponse []struct { + GroupID string `json:"groupId"` + } + if r, err := c.client.R(). + SetResult(&groupsResponse). + Get("/group/list"); err != nil { + return []string{}, err + } else { + if r.IsError() { + return []string{}, fmt.Errorf("error fetching groups: (%s) %s", r.Status(), r.Body()) + } + var result []string + for _, g := range groupsResponse { + result = append(result, g.GroupID) + } + return result, nil + } +} + +func (c CloudianAPI) GetBucketSize(groupID string, userID string, bucket string) (int64, error) { + var bucketByteCount int64 + if r, err := c.client.R(). + SetQueryParam("groupId", groupID). + SetQueryParam("userId", userID). + SetQueryParam("bucketName", bucket). + SetResult(&bucketByteCount). + Get("/system/bytecount"); err != nil { + return 0, err + } else { + if r.IsError() { + return 0, fmt.Errorf("error fetching bucket size from Cloudian group %s, user %s, bucket %s: (%s) %s", groupID, userID, bucket, r.Status(), r.Body()) + } + return bucketByteCount, nil + } +} diff --git a/internal/exporter.go b/internal/exporter.go new file mode 100644 index 0000000..4ce2717 --- /dev/null +++ b/internal/exporter.go @@ -0,0 +1,69 @@ +package internal + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "time" +) + +type metric struct { + size *prometheus.GaugeVec +} + +type CloudianExporter struct { + api CloudianAPI + refresh time.Duration + exitChannel chan bool + metric *metric +} + +func NewCloudianExporter(refresh time.Duration, api CloudianAPI, exitChannel chan bool, registry *prometheus.Registry) CloudianExporter { + m := &metric{ + size: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "cloudian_bucket_size", + Help: "Size of bucket in bytes", + }, []string{"group_id", "user_id", "bucket"}), + } + registry.MustRegister(m.size) + return CloudianExporter{ + api: api, + refresh: refresh, + exitChannel: exitChannel, + metric: m, + } +} + +func (e CloudianExporter) Run() { + go func() { + for { + select { + case <-e.exitChannel: + return + default: + logrus.Debug("Fetching metrics") + if r, err := e.api.GetGroups(); err != nil { + logrus.Errorf("Error fetching groups: %v", err) + } else { + for _, groupID := range r { + if b, err := e.api.GetBuckets(groupID); err != nil { + logrus.Errorf("Error fetching buckets for group %s: %v", groupID, err) + } else { + for _, userBucket := range b { + if s, err := e.api.GetBucketSize(groupID, userBucket.UserID, userBucket.Bucket); err != nil { + logrus.Errorf("Error fetching bucket size for group %s, user %s, bucket %s: %v", groupID, userBucket.UserID, userBucket.Bucket, err) + } else { + e.metric.size.With(prometheus.Labels{ + "group_id": groupID, + "user_id": userBucket.UserID, + "bucket": userBucket.Bucket, + }).Set(float64(s)) + } + } + } + } + } + time.Sleep(e.refresh) + } + } + }() +}