feat(ckan): Backup updates (#472)

* feat(ckan): Add backup logic * feat(ckan): Add backup logic * feat(ckan): Add beaker session secret * feat(cron): Correct cron to use latest spec * feat(deploy): Not sure this is needed * feat(ckan): Add spec
StatCan · Dec 11, 2024 · 39c1580 · 39c1580
1 parent 9cb3d91
commit 39c1580
Show file tree

Hide file tree

Showing 6 changed files with 70 additions and 6 deletions.
diff --git a/stable/ckan/conf/ckan/overlay-production.ini b/stable/ckan/conf/ckan/overlay-production.ini
@@ -18,9 +18,16 @@
 #
 
 [app:main]
-ckan.site_title = CKAN-SCHEMING-FDI
+ckan.locales_offered = en fr
 ckan.site_logo = /base/images/ckan-logo.png
 ckan.site_description =
+ckan.auth.user_create_organizations = False
+ckan.auth.user_create_groups = False
+ckan.auth.user_delete_groups = false
+ckan.auth.user_delete_organizations = false
+ckan.auth.create_dataset_if_not_in_organization = False
+ckan.auth.create_unowned_dataset = False
+ckan.auth.public_activity_stream_detail = true
 ckan.favicon = /base/images/ckan.ico
 ckan.gravatar_default = identicon
 ckan.preview.direct = png jpg gif

diff --git a/stable/ckan/templates/cm/ckan-crontab.yaml b/stable/ckan/templates/cm/ckan-crontab.yaml
@@ -0,0 +1,13 @@
+{{- if .Values.ckan.backup.enabled }}
+{{- $fullName := include "ckan.fullname" . -}}
+{{- $adminuser := .Values.ckan.backup.adminuser -}}
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: {{ include "ckan.fullname" . }}-crontabs
+data:
+{{- range $key, $value := .Values.ckan.cronjobs }}
+  {{ $key }}.sh: |
+    {{ $value.command }} -c /srv/app/production.ini -O /app/backup/{{ $fullName }}-{{ $key }}-$(date +%Y-%m-%d).json1.gz -z -u {{ $adminuser }} >> /app/backup/{{ $fullName }}-{{ $key }}-$(date +%Y-%m-%d)-log.txt ;
+{{- end }}
+{{- end }}
diff --git a/stable/ckan/templates/cronjob/ckan.yaml b/stable/ckan/templates/cronjob/ckan.yaml
@@ -1,9 +1,5 @@
-{{- if eq .Values.ckan.activityStreamsEmailNotifications_jobs "true" -}}
-{{- if semverCompare "<1.21" .Capabilities.KubeVersion.GitVersion }}
-apiVersion: batch/v1beta1
-{{- else }}
+{{- if .Values.ckan.activityStreamsEmailNotifications_jobs -}}
 apiVersion: batch/v1
-{{- end }}
 kind: CronJob
 metadata:
   name: {{ include "ckan.name" . }}-email-notifications

diff --git a/stable/ckan/templates/deploy/ckan.yaml b/stable/ckan/templates/deploy/ckan.yaml
@@ -33,6 +33,16 @@ spec:
       securityContext:
         {{- toYaml .Values.ckan.podSecurityContext | nindent 8 }}
       volumes:
+        {{- if .Values.ckan.backup.enabled }}
+        - name: "ckan-backup"
+          azureFile:
+            secretName: {{ include "ckan.fullname" . }}-ckan-backup-secret
+            shareName: {{ .Values.ckan.backup.shareName }}
+        - name: {{ include "ckan.fullname" . }}-crontabs
+          configMap:
+            name: {{ include "ckan.fullname" . }}-crontabs
+            defaultMode: 0755
+        {{- end }}
         - name: {{ include "ckan.fullname" . }}-ini-overlay-configmap
           configMap:
             name: {{ include "ckan.fullname" . }}-ini-overlay-configmap
@@ -241,6 +251,14 @@ spec:
           - |
             set -e
             source $CKAN_VENV/bin/activate
+            sudo service cron start
+        {{- if .Values.ckan.backup.enabled }}
+            echo "#!/usr/bin/bash" | crontab -
+            (crontab -l ; echo "PATH=/usr/lib/ckan/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") | crontab -
+        {{- range $key, $value := .Values.ckan.cronjobs }}
+            (crontab -l ; echo {{ $value.crontab | quote }}) | crontab -
+        {{- end }}
+        {{- end }}
             ckan generate config /srv/app/production.ini
             python3 <<EOF | xargs -I{} ckan config-tool /srv/app/production.ini "{}"
             from ckan.config.environment import CONFIG_FROM_ENV_VARS
@@ -254,6 +272,8 @@ spec:
             echo "Applying the production.ini overlay..."
             ckan config-tool /srv/app/production.ini -f /config/production.ini
             ckan -c /srv/app/production.ini db init
+            printenv > /tmp/enviroment
+            sudo bash -c "cat /tmp/enviroment > /etc/environment"
             python3 /srv/prerun/prerun.py
             ckan -c /srv/app/production.ini run --host 0.0.0.0
           ports:
@@ -296,6 +316,8 @@ spec:
             - name: TRANSFER_API_VERSION
               value: {{ .Values.extensions.sdmx.transferApiVersion }}
             {{- end }}
+            - name: BEAKER_SESSION_SECRET
+              value: {{ .Values.ckan.beaker_session_secret }}
             - name: CKAN_CONFIG
               value: /srv/app/
             - name: CKAN_SYSADMIN_NAME
@@ -410,6 +432,12 @@ spec:
             failureThreshold: {{ .Values.ckan.liveness.failureThreshold }}
             timeoutSeconds: {{ .Values.ckan.liveness.timeoutSeconds }}
           volumeMounts:
+            {{- if .Values.ckan.backup.enabled}}
+          - mountPath: "/app/backup"
+            name: "ckan-backup"
+          - name: {{ include "ckan.fullname" . }}-crontabs
+            mountPath: /app/
+            {{- end }}
           - name: {{ include "ckan.fullname" . }}-prerun-configmap
             mountPath: /srv/prerun
           - name: {{ include "ckan.fullname" . }}-ini-overlay-configmap

diff --git a/stable/ckan/templates/secret/ckan-backup-secret.yaml b/stable/ckan/templates/secret/ckan-backup-secret.yaml
@@ -0,0 +1,10 @@
+{{- if .Values.ckan.backup.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "ckan.fullname" . }}-ckan-backup-secret
+type: Opaque
+data:
+  azurestorageaccountname: {{ .Values.ckan.backup.azurestorageaccountname | b64enc }}
+  azurestorageaccountkey: {{ .Values.ckan.backup.azurestorageaccountkey | b64enc }}
+{{- end }}
diff --git a/stable/ckan/values.yaml b/stable/ckan/values.yaml
@@ -69,6 +69,12 @@ ckan:
   activityStreamsEmailNotifications_jobs: "false"
   debug: "false"
   maintenanceMode: "false"
+  backup:
+    adminuser: ""
+    enabled: true
+    shareName: ""
+    azurestorageaccountname: ""
+    azurestorageaccountkey: ""
   psql:
     initialize: true
   db:
@@ -157,6 +163,10 @@ ckan:
     name:
 
 ## Configuration values for DataPusher
+datapusher:
+  image:
+    repository: artifactory.cloud.statcan.ca/docker/keitaro/ckan-datapusher
+    tag: 0.0.17
 
 ## Configuration values for Solr
 ## ToDo: Switch to Solr Operator sub-chart