From c4c800cbaa9d08b3dae9b992e2eefcc2bae8b422 Mon Sep 17 00:00:00 2001 From: Jan Max Tiedemann Date: Fri, 28 Jun 2024 10:21:02 +0200 Subject: [PATCH] feat: remove rclone --- .github/workflows/helm-publish.yaml | 2 +- charts/rclone-copy/.helmignore | 2 - charts/rclone-copy/Chart.yaml | 8 -- charts/rclone-copy/README.md | 74 ---------------- charts/rclone-copy/templates/_helpers.tpl | 29 ------ charts/rclone-copy/templates/configmap.yaml | 9 -- charts/rclone-copy/templates/rclone-cron.yaml | 88 ------------------- charts/rclone-copy/values.yaml | 78 ---------------- 8 files changed, 1 insertion(+), 289 deletions(-) delete mode 100644 charts/rclone-copy/.helmignore delete mode 100644 charts/rclone-copy/Chart.yaml delete mode 100644 charts/rclone-copy/README.md delete mode 100644 charts/rclone-copy/templates/_helpers.tpl delete mode 100644 charts/rclone-copy/templates/configmap.yaml delete mode 100644 charts/rclone-copy/templates/rclone-cron.yaml delete mode 100644 charts/rclone-copy/values.yaml diff --git a/.github/workflows/helm-publish.yaml b/.github/workflows/helm-publish.yaml index f6765509..84a81eff 100644 --- a/.github/workflows/helm-publish.yaml +++ b/.github/workflows/helm-publish.yaml @@ -35,7 +35,7 @@ jobs: needs: get-gradle-version with: charts-path: "./charts" - subdirs: "['producer-app','producer-app-cleanup-job','rclone-copy','streams-app','streams-app-cleanup-job']" + subdirs: "['producer-app','producer-app-cleanup-job','streams-app','streams-app-cleanup-job']" gh-pages-branch: gh-pages version: ${{ needs.get-gradle-version.outputs.version }} secrets: diff --git a/charts/rclone-copy/.helmignore b/charts/rclone-copy/.helmignore deleted file mode 100644 index a1006ff6..00000000 --- a/charts/rclone-copy/.helmignore +++ /dev/null @@ -1,2 +0,0 @@ -*.tgz -.* diff --git a/charts/rclone-copy/Chart.yaml b/charts/rclone-copy/Chart.yaml deleted file mode 100644 index 1ef2c031..00000000 --- a/charts/rclone-copy/Chart.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -name: rclone-copy -description: A helm chart for rclone copy on Kubernetes. -version: 1.0.0 -maintainers: - - name: bakdata - email: opensource@bakdata.com - url: bakdata.com diff --git a/charts/rclone-copy/README.md b/charts/rclone-copy/README.md deleted file mode 100644 index df72a355..00000000 --- a/charts/rclone-copy/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# rclone Helm for Kubernetes - -## Usage: -- Create `rclone.conf` (either via `rclone configure` or from `rclone.conf.template`) -- Adapt `values.yaml` to contain your values. -- Install helm with: `helm install . -f values.yaml -n "mySource-mySoruceType-myDestType"` in this directory. - -## Creating rclone.conf from scratch -- Download `rclone` (on Mac e.g. `brew install rclone`) -- Run the following commands: -```bash -cd my-repository -rclone config --config "rclone.conf" -``` -- Select `n` for "New remote" -- Enter name of source. The source name what you call the rclone copy command with. -- Select `10` for `ftp`. -- Enter ftp URL. Should not contain `ftp://`! -- Enter username. -- Enter port if different than 21. Otherwise just ENTER. -- Select `y` to add password. -- Add password twice. -- Select `n` to skip advanced settings. -- Select `y` to finish this config. - -Now step one is done. Next we need to add S3. -- Select `n` for new remote. -- The name should be "s3" (without quotes) -- Select `4` for AWS-compliant sources and then `1` for AWS as the provider. -- Select `2` to use credentials from the environment (we use IAM) -- Press ENTER to leave credentials blank. -- Press ENTER to leave access key blank. -- Select `9` for EU Frankfurt region. -- Press ENTER to to leave default endpoint. -- Select `9` for EU region. -- Press ENTER to leave default ACL (== private). -- Press ENTER to leave default server-side encryption. -- Press ENTER to leave default ARN. -- Press ENTER to leave default storage class. -- Select `n` to skip advanced settings. -- Select `y` to finish source. -- Select `q` to quit config and verfiy that the config is correct. - -Your config file should look like this now: -``` -[my-source] -type = ftp -host = ftp.my-source.com -user = my-user -pass = 1FbNBhe7ndNsFcjTVofIarv1oP3d - -[s3] -type = s3 -provider = AWS -env_auth = true -region = eu-central-1 -location_constraint = EU -``` - -Now we need to encrypt the file. -- `rclone config --config "rclone.conf"` to open config again. -- Select `s` to set configuration password. -- Select `a` to add own password. -- Enter the password twice. -- Select `q` to finish encryption. -- Select `q` to close config menu. - -The config file is now complete and should look something like this: -``` -# Encrypted rclone configuration File - -RCLONE_ENCRYPT_V0: -nUebCNSIfjPZ3484t4Q2iSm252SkPCbMgheI1b//xegm0Hv2Fb6d4SkjgnGii6jy+dVK9f9NVhdoEcG/o4g4XHTdpHKl8GzDwhY90wZZIpNH1ID45XSVeDvRWAJSbqiLtoss4gEfCVCf/QvPWbcPon6ZGFGN8XszH2S7wQhnjL2Uwl1BrE6NwRurvoZ+Hsp6ml1E1mny8iI1WGopLPjkqJCk5GT6nJmy2YyLZbOsO7bs4rJy0p3GM4KBw6VuTza4+WKWRlJweKFo3gHB85Xa8xbBgpznOAx+MUfTKXyvkmMkZlVc37tC0rv4 -``` diff --git a/charts/rclone-copy/templates/_helpers.tpl b/charts/rclone-copy/templates/_helpers.tpl deleted file mode 100644 index 4028b9cf..00000000 --- a/charts/rclone-copy/templates/_helpers.tpl +++ /dev/null @@ -1,29 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "rclone-copy.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "rclone-copy.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Values.nameOverride -}} -{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "rclone-copy.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/charts/rclone-copy/templates/configmap.yaml b/charts/rclone-copy/templates/configmap.yaml deleted file mode 100644 index 6255f672..00000000 --- a/charts/rclone-copy/templates/configmap.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: rclone-config-{{ .Release.Name }} -data: - rclone.conf: | -{{ .Values.rcloneConf | indent 4 }} - include-pattern.conf: | -{{ .Values.includePattern | indent 4 }} diff --git a/charts/rclone-copy/templates/rclone-cron.yaml b/charts/rclone-copy/templates/rclone-cron.yaml deleted file mode 100644 index 7ddf601c..00000000 --- a/charts/rclone-copy/templates/rclone-cron.yaml +++ /dev/null @@ -1,88 +0,0 @@ -{{- $root := . -}} -{{- if .Capabilities.APIVersions.Has "batch/v1/CronJob" }} -apiVersion: batch/v1 -{{- else }} -apiVersion: batch/v1beta1 -{{- end }} -kind: CronJob -metadata: - name: {{ template "rclone-copy.fullname" . }} -{{- if .Values.annotations }} - annotations: - {{- range $key, $value := .Values.annotations }} - {{ $key | quote }}: {{ $value | quote }} - {{- end }} -{{- end }} - labels: - app: {{ template "rclone-copy.name" . }} - chart: {{ template "rclone-copy.chart" . }} - release: {{ .Release.Name }} - {{- range $key, $value := .Values.labels }} - {{ $key }}: {{ $value }} - {{- end }} -spec: - suspend: {{ .Values.suspend }} - schedule: "{{ .Values.schedule }}" - successfulJobsHistoryLimit: {{ .Values.successfulJobsHistoryLimit }} - failedJobsHistoryLimit: {{ .Values.failedJobsHistoryLimit }} - concurrencyPolicy: Replace - jobTemplate: - spec: - template: - metadata: - {{- if .Values.podAnnotations }} - annotations: - {{- range $key, $value := .Values.podAnnotations }} - {{ $key | quote }}: {{ $value | quote }} - {{- end }} - {{- end }} - labels: - app: {{ template "rclone-copy.name" . }} - release: {{ .Release.Name }} - {{- range $key, $value := .Values.podLabels }} - {{ $key }}: {{ $value }} - {{- end }} - spec: - {{- if .Values.serviceAccountName }} - serviceAccountName: {{ .Values.serviceAccountName }} - {{- end }} - {{- if .Values.tolerations }} - tolerations: -{{ toYaml .Values.tolerations | indent 12 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- tpl (toYaml .) $root | nindent 12 }} - {{- end }} - containers: - - name: rclone-container - image: rclone/rclone:{{ .Values.imageRelease }} - - command: - - /bin/sh - args: - - -c - # copy as workaround for rclone.conf read only (see https://github.com/rclone/rclone/issues/3655) - - >- - cp /root/.config/rclone/rclone_ro.conf /root/.config/rclone/rclone.conf && - rclone copy -v {{ .Values.arguments | join " " }} --include-from /root/include-pattern.conf "{{ .Values.sync.source.name }}:{{ .Values.sync.source.path }}" "{{ .Values.sync.dest.name }}:{{ .Values.sync.dest.path }}" - - volumeMounts: - - name: config - # This is the default path where the rclone implementation assumes the config is located - mountPath: "/root/.config/rclone/rclone_ro.conf" - subPath: "rclone.conf" - - name: config - mountPath: "/root/include-pattern.conf" - subPath: "include-pattern.conf" - - resources: -{{ toYaml .Values.resources | indent 14 }} - - restartPolicy: {{ .Values.restartPolicy }} - volumes: - - name: config - configMap: - name: rclone-config-{{ .Release.Name }} - backoffLimit: {{ .Values.backoffLimit }} - diff --git a/charts/rclone-copy/values.yaml b/charts/rclone-copy/values.yaml deleted file mode 100644 index ace2fc06..00000000 --- a/charts/rclone-copy/values.yaml +++ /dev/null @@ -1,78 +0,0 @@ -nameOverride: bakdata-rclone-copy - -# This is the cofiguration for the specific rclone-cronjob that you want to run. -# `sync` contains the config for the source and destination of the rclone copy job. -# It can be interpreted as: -# rclone copy {source.name}:{source.path} {dest.name}:{dest.path} -sync: - source: - # This is the name for the source remote from the rclone.conf file. - # If the value specified here is not present in the config file, the job will fail. - # See rclone.conf.template for an example. - name: my-sftp - - # This is the path to the source directory. - path: /tmp/sync-dir - - dest: - # This is the name for the destination remote from the rclone.conf file. - # If the value specified here is not present in the config file, the job will fail. - # See rclone.conf.template for an example. - name: my-s3 - - # This is the path to the target directory. - path: my-s3-bucket - -# Additional arguments to pass to rclone. -# The expected value is an array with comma seperated flag and value passed as a string. -# Example: ["--transfers", '"1"'] -arguments: [] - -# Pattern of files to copy. Standard UNIX file glob patterns are used. Default: "*" -# See templates/include-pattern-config.yaml for more details on usage in ConfigMap. -includePattern: "*" - -# Cron schedule for this connect job. Default is at 12:00 every day. -schedule: "0 12 * * *" - -suspend: false - -# Content of the rclone.conf file. -rcloneConf: "" - -# Release version for the openbridge/ob_bulkstash docker image. -# You probably don't need to change this. -imageRelease: 1.53.1 - -restartPolicy: OnFailure - -# serviceAccountName: foo - -tolerations: [] -# - key: "foo" -# operator: "Exists" -# effect: "NoSchedule" -# - key: "bar" -# operator: "Exists" -# effect: "NoSchedule" - -## Affinity for pod assignment (evaluated as template) -## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity -## -affinity: {} - -resources: - requests: - cpu: 200m - memory: 300Mi - limits: - memory: 2G - cpu: 500m - -successfulJobsHistoryLimit: 1 -failedJobsHistoryLimit: 1 -backoffLimit: 6 - -podAnnotations: {} - -podLabels: {}