Move rclone from streams-bootstrap

.bumpversion.cfg

@@ -0,0 +1,2 @@
+[bumpversion]
+current_version = 0.0.1

.github/lint-config.yaml

@@ -0,0 +1 @@
+target-branch: "main"

.github/workflows/helm-lint.yaml

@@ -0,0 +1,14 @@
+name: Helm lint
+
+on:
+  push:
+
+jobs:
+  helm-lint:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Lint Helm chart
+        uses: bakdata/ci-templates/actions/[email protected]
+        with:
+          lint-config-path: ".github/lint-config.yaml"
+          ref: ${{ github.ref_name }}

.github/workflows/helm-publish.yaml

@@ -0,0 +1,38 @@
+name: Publish Helm Charts
+
+on:
+  push:
+
+jobs:
+  get-version:
+    name: Get version from bumpversion
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.get-version.outputs.version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Get current version from bumpversion
+        id: get-version
+        run: |
+          version=$(cat .bumpversion.cfg | sed -nE 's/.*current_version = (.*)/\1/p')
+          if [[ "${GITHUB_REF#refs/heads/}" != "main" && "${GITHUB_REF#refs/heads/}" != "master" ]]; then
+            version="${version}-SNAPSHOT"
+          fi
+          echo $version
+          echo "version=$version" >> $GITHUB_OUTPUT
+        shell: bash
+
+  call-workflow-passing-data:
+    name: Publish Helm chart
+    uses: bakdata/ci-templates/.github/workflows/[email protected]
+    needs: get-version
+    with:
+      charts-path: "./charts"
+      subdirs: "['rclone-copy']"
+      gh-pages-branch: gh-pages
+      version: ${{ needs.get-version.outputs.version }}
+    secrets:
+      github-username: ${{ secrets.GH_USERNAME }}
+      github-email: ${{ secrets.GH_EMAIL }}
+      github-token: ${{ secrets.GH_TOKEN }}

.github/workflows/release.yaml

@@ -0,0 +1,25 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      release-type:
+        description: "The scope of the release (major, minor or patch)."
+        type: choice
+        required: true
+        default: patch
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  bump-version-release:
+    name: Bumpversion Release
+    uses: bakdata/ci-templates/.github/workflows/[email protected]
+    with:
+      release-type: "${{ inputs.release-type }}"
+    secrets:
+      github-email: "${{ secrets.GH_EMAIL }}"
+      github-username: "${{ secrets.GH_USERNAME }}"
+      github-token: "${{ secrets.GH_TOKEN }}"

charts/.helmignore

@@ -0,0 +1,2 @@
+*.tgz
+.*

charts/rclone-copy/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+name: rclone-copy
+description: A helm chart for rclone copy on Kubernetes.
+version: 1.0.0
+maintainers:
+  - name: bakdata
+    email: [email protected]
+    url: bakdata.com

charts/rclone-copy/README.md

@@ -0,0 +1,74 @@
+# rclone Helm for Kubernetes
+
+## Usage:
+- Create `rclone.conf` (either via `rclone configure` or from `rclone.conf.template`)
+- Adapt `values.yaml` to contain your values.
+- Install helm with: `helm install . -f values.yaml -n "mySource-mySoruceType-myDestType"` in this directory.
+
+## Creating rclone.conf from scratch
+- Download `rclone` (on Mac e.g. `brew install rclone`)
+- Run the following commands:
+```bash 
+cd my-repository
+rclone config --config "rclone.conf"
+```
+- Select `n` for "New remote"
+- Enter name of source. The source name what you call the rclone copy command with.
+- Select `10` for `ftp`.
+- Enter ftp URL. Should not contain `ftp://`!
+- Enter username.
+- Enter port if different than 21. Otherwise just ENTER.
+- Select `y` to add password.
+- Add password twice.
+- Select `n` to skip advanced settings.
+- Select `y` to finish this config.
+
+Now step one is done. Next we need to add S3.
+- Select `n` for new remote.
+- The name should be "s3" (without quotes)
+- Select `4` for AWS-compliant sources and then `1` for AWS as the provider.
+- Select `2` to use credentials from the environment (we use IAM)
+- Press ENTER to leave credentials blank.
+- Press ENTER to leave access key blank.
+- Select `9` for EU Frankfurt region.
+- Press ENTER to to leave default endpoint.
+- Select `9` for EU region.
+- Press ENTER to leave default ACL (== private).
+- Press ENTER to leave default server-side encryption.
+- Press ENTER to leave default ARN.
+- Press ENTER to leave default storage class.
+- Select `n` to skip advanced settings.
+- Select `y` to finish source.
+- Select `q` to quit config and verfiy that the config is correct.
+
+Your config file should look like this now:
+```
+[my-source]
+type = ftp
+host = ftp.my-source.com
+user = my-user
+pass = 1FbNBhe7ndNsFcjTVofIarv1oP3d
+
+[s3]
+type = s3
+provider = AWS
+env_auth = true
+region = eu-central-1
+location_constraint = EU
+```
+
+Now we need to encrypt the file.
+- `rclone config --config "rclone.conf"` to open config again.
+- Select `s` to set configuration password.
+- Select `a` to add own password.
+- Enter the password twice.
+- Select `q` to finish encryption.
+- Select `q` to close config menu.
+
+The config file is now complete and should look something like this:
+```
+# Encrypted rclone configuration File
+
+RCLONE_ENCRYPT_V0:
+nUebCNSIfjPZ3484t4Q2iSm252SkPCbMgheI1b//xegm0Hv2Fb6d4SkjgnGii6jy+dVK9f9NVhdoEcG/o4g4XHTdpHKl8GzDwhY90wZZIpNH1ID45XSVeDvRWAJSbqiLtoss4gEfCVCf/QvPWbcPon6ZGFGN8XszH2S7wQhnjL2Uwl1BrE6NwRurvoZ+Hsp6ml1E1mny8iI1WGopLPjkqJCk5GT6nJmy2YyLZbOsO7bs4rJy0p3GM4KBw6VuTza4+WKWRlJweKFo3gHB85Xa8xbBgpznOAx+MUfTKXyvkmMkZlVc37tC0rv4
+```

charts/rclone-copy/templates/_helpers.tpl

@@ -0,0 +1,29 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "rclone-copy.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "rclone-copy.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Values.nameOverride -}}
+{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "rclone-copy.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}

charts/rclone-copy/templates/configmap.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rclone-config-{{ .Release.Name }}
+data:
+  rclone.conf: |
+{{ .Values.rcloneConf | indent 4 }}
+  include-pattern.conf: |
+{{ .Values.includePattern | indent 4 }}

charts/rclone-copy/templates/rclone-cron.yaml

@@ -0,0 +1,88 @@
+{{- $root := . -}}
+{{- if .Capabilities.APIVersions.Has "batch/v1/CronJob" }}
+apiVersion: batch/v1
+{{- else }}
+apiVersion: batch/v1beta1
+{{- end }}
+kind: CronJob
+metadata:
+  name: {{ template "rclone-copy.fullname" . }}
+{{- if .Values.annotations }}
+  annotations:
+  {{- range $key, $value := .Values.annotations }}
+    {{ $key | quote }}: {{ $value | quote }}
+  {{- end }}
+{{- end }}
+  labels:
+    app: {{ template "rclone-copy.name" . }}
+    chart: {{ template "rclone-copy.chart" . }}
+    release: {{ .Release.Name }}
+    {{- range $key, $value := .Values.labels }}
+    {{ $key }}: {{ $value }}
+    {{- end }}
+spec:
+  suspend: {{ .Values.suspend }}
+  schedule: "{{ .Values.schedule }}"
+  successfulJobsHistoryLimit: {{ .Values.successfulJobsHistoryLimit }}
+  failedJobsHistoryLimit: {{ .Values.failedJobsHistoryLimit }}
+  concurrencyPolicy: Replace
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+        {{- if .Values.podAnnotations }}
+          annotations:
+          {{- range $key, $value := .Values.podAnnotations }}
+            {{ $key | quote }}: {{ $value | quote }}
+          {{- end }}
+        {{- end }}
+          labels:
+            app: {{ template "rclone-copy.name" . }}
+            release: {{ .Release.Name }}
+            {{- range $key, $value := .Values.podLabels }}
+            {{ $key }}: {{ $value }}
+            {{- end }}
+        spec:
+          {{- if .Values.serviceAccountName }}
+          serviceAccountName: {{ .Values.serviceAccountName }}
+          {{- end }}
+          {{- if .Values.tolerations }}
+          tolerations:
+{{ toYaml .Values.tolerations | indent 12 }}
+          {{- end }}
+          {{- with .Values.affinity }}
+          affinity:
+            {{- tpl (toYaml .) $root | nindent 12 }}
+          {{- end }}
+          containers:
+          - name: rclone-container
+            image: rclone/rclone:{{ .Values.imageRelease }}
+
+            command:
+              - /bin/sh
+            args:
+              - -c
+              # copy as workaround for rclone.conf read only (see https://github.com/rclone/rclone/issues/3655)
+              - >-
+                cp /root/.config/rclone/rclone_ro.conf /root/.config/rclone/rclone.conf &&
+                rclone copy -v {{ .Values.arguments | join " " }} --include-from /root/include-pattern.conf "{{ .Values.sync.source.name }}:{{ .Values.sync.source.path }}" "{{ .Values.sync.dest.name }}:{{ .Values.sync.dest.path }}"
+
+            volumeMounts:
+              - name: config
+                # This is the default path where the rclone implementation assumes the config is located
+                mountPath: "/root/.config/rclone/rclone_ro.conf"
+                subPath: "rclone.conf"
+              - name: config
+                mountPath: "/root/include-pattern.conf"
+                subPath: "include-pattern.conf"
+
+            resources:
+{{ toYaml .Values.resources | indent 14 }}
+
+          restartPolicy: {{ .Values.restartPolicy }}
+          volumes:
+            - name: config
+              configMap:
+                name: rclone-config-{{ .Release.Name }}
+      backoffLimit: {{ .Values.backoffLimit }}
+

charts/rclone-copy/values.yaml

@@ -0,0 +1,78 @@
+nameOverride: bakdata-rclone-copy
+
+# This is the cofiguration for the specific rclone-cronjob that you want to run.
+# `sync` contains the config for the source and destination of the rclone copy job.
+# It can be interpreted as:
+# rclone copy {source.name}:{source.path} {dest.name}:{dest.path}
+sync:
+  source:
+    # This is the name for the source remote from the rclone.conf file.
+    # If the value specified here is not present in the config file, the job will fail.
+    # See rclone.conf.template for an example.
+    name: my-sftp
+
+    # This is the path to the source directory.
+    path: /tmp/sync-dir
+
+  dest:
+    # This is the name for the destination remote from the rclone.conf file.
+    # If the value specified here is not present in the config file, the job will fail.
+    # See rclone.conf.template for an example.
+    name: my-s3
+
+    # This is the path to the target directory.
+    path: my-s3-bucket
+
+# Additional arguments to pass to rclone.
+# The expected value is an array with comma seperated flag and value passed as a string.
+# Example: ["--transfers", '"1"']
+arguments: []
+
+# Pattern of files to copy. Standard UNIX file glob patterns are used. Default: "*"
+# See templates/include-pattern-config.yaml for more details on usage in ConfigMap.
+includePattern: "*"
+
+# Cron schedule for this connect job. Default is at 12:00 every day.
+schedule: "0 12 * * *"
+
+suspend: false
+
+# Content of the rclone.conf file.
+rcloneConf: ""
+
+# Release version for the openbridge/ob_bulkstash docker image.
+# You probably don't need to change this.
+imageRelease: 1.53.1
+
+restartPolicy: OnFailure
+
+# serviceAccountName: foo
+
+tolerations: []
+#   - key: "foo"
+#     operator: "Exists"
+#     effect: "NoSchedule"
+#   - key: "bar"
+#     operator: "Exists"
+#     effect: "NoSchedule"
+
+## Affinity for pod assignment (evaluated as template)
+## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+##
+affinity: {}
+
+resources:
+  requests:
+    cpu: 200m
+    memory: 300Mi
+  limits:
+    memory: 2G
+    cpu: 500m
+
+successfulJobsHistoryLimit: 1
+failedJobsHistoryLimit: 1
+backoffLimit: 6
+
+podAnnotations: {}
+
+podLabels: {}