Merge branch 'master' into add_tpv

galaxyproject · Aug 9, 2022 · fb772aa · fb772aa
2 parents ce51c15 + 6175938
commit fb772aa
Show file tree

Hide file tree

Showing 17 changed files with 199 additions and 262 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -1,4 +1,4 @@
-name: Linting and deployment tests (w/ & w/out CVMFS) on K3S
+name: Linting and deployment test on K3S
 on:
   push:
     branches:
@@ -33,7 +33,8 @@ jobs:
       - name: Start k8s locally
         uses: jupyterhub/action-k3s-helm@v1
         with:
-          k3s-version: v1.19.10+k3s1 # releases:  https://github.com/k3s-io/k3s/tags
+          k3s-version: v1.24.3+k3s1  # releases:  https://github.com/k3s-io/k3s/tags
+          metrics-enabled: false
           traefik-enabled: false
       - name: Verify function of k8s, kubectl, and helm
         run: |
@@ -46,7 +47,7 @@ jobs:
       - name: Helm repo add galaxy
         run: helm repo add galaxy https://github.com/CloudVE/helm-charts/raw/master
       - name: Helm install Galaxy
-        run: time bash -c 'helm install --create-namespace -n galaxy galaxy ./galaxy --set persistence.accessMode="ReadWriteOnce" --set persistence.size="5Gi" --set postgresql.deploy=true --set resources.requests.memory=0Mi,resources.requests.cpu=0m --wait --timeout=600s'
+        run: time bash -c 'helm install --create-namespace -n galaxy galaxy ./galaxy --set persistence.accessMode="ReadWriteOnce" --set resources.requests.memory=0Mi,resources.requests.cpu=0m --wait --timeout=600s'
       - name: Get events
         run: kubectl get events -n galaxy; kubectl get events -n csi-drivers;
         if: always()
@@ -59,43 +60,3 @@ jobs:
       - name: Print workflow handler log
         run: bash -c "kubectl logs -n galaxy $(kubectl -n galaxy get pods | grep -o '[^ ]*galaxy-workflow[^ ]*')"
         if: always()
-  cvmfstest:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Install Helm
-        run: curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
-      - uses: actions/checkout@v2
-        with:
-          persist-credentials: false
-      - name: Helm dep update
-        run: cd galaxy/ && helm dep update && cd ..
-      - name: Start k8s locally
-        uses: jupyterhub/action-k3s-helm@v1
-        with:
-          k3s-version: v1.19.10+k3s1 # releases:  https://github.com/k3s-io/k3s/tags
-
-      - name: Verify function of k8s, kubectl, and helm
-        run: |
-          echo "kubeconfig: $KUBECONFIG"
-          kubectl version
-          kubectl get pods --all-namespaces
-
-          helm version
-          helm list
-      - name: Helm repo add galaxy
-        run: helm repo add galaxy https://github.com/CloudVE/helm-charts/raw/master
-      - name: Helm install Galaxy
-        run: time bash -c 'helm install --create-namespace -n galaxy galaxy ./galaxy --set persistence.accessMode="ReadWriteOnce" --set persistence.size="5Gi" --set cvmfs.enabled=true --set cvmfs.deploy=true --set cvmfs.cache.preload.enabled=false --set cvmfs.cache.alienCache.enabled=false --set cvmfs.cache.localCache.enabled=true --set postgresql.deploy=true --set resources.requests.memory=0Mi,resources.requests.cpu=0m --set setupJob.downloadToolConfs.enabled=false --set cvmfs.repositories.cvmfs-gxy-cloud=cloud.galaxyproject.org --set cvmfs.galaxyPersistentVolumeClaims.cloud.storage=1Gi --set cvmfs.galaxyPersistentVolumeClaims.cloud.storageClassName=cvmfs-gxy-cloud --set cvmfs.galaxyPersistentVolumeClaims.cloud.mountPath=/cvmfs/cloud.galaxyproject.org --wait --timeout=600s'
-      - name: Get events
-        run: kubectl get events -n galaxy; kubectl get events -n csi-drivers
-        if: always()
-      - name: Print web handler log
-        run: bash -c "kubectl logs -n galaxy $(kubectl -n galaxy get pods | grep -o '[^ ]*galaxy-web[^ ]*')"
-        if: always()
-      - name: Print job handler log
-        run: bash -c "kubectl logs -n galaxy $(kubectl -n galaxy get pods | grep -o '[^ ]*galaxy-job[^ ]*')"
-        if: always()
-      - name: Print workflow handler log
-        run: bash -c "kubectl logs -n galaxy $(kubectl -n galaxy get pods | grep -o '[^ ]*galaxy-workflow[^ ]*')"
-        if: always()
-
diff --git a/README.md b/README.md
diff --git a/galaxy/Chart.yaml b/galaxy/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: galaxy
 type: application
-version: 4.10.2
+version: 5.0.0
 appVersion: "21.09"
 description: Chart for Galaxy, an open, web-based platform for accessible, reproducible, and transparent computational biomedical research.
 icon: https://galaxyproject.org/images/galaxy-logos/galaxy_project_logo_square.png
@@ -20,3 +20,10 @@ dependencies:
     alias: cvmfs
     tags:
       - deploy-cvmfs
+  - name: csi-s3
+    repository: https://raw.githubusercontent.com/cloudve/helm-charts/master/
+    version: 0.31.3
+    condition: s3csi.deploy
+    alias: s3csi
+    tags:
+      - deploy-s3csi
diff --git a/galaxy/files/configs/tool_conf.xml b/galaxy/files/configs/tool_conf.xml
@@ -20,6 +20,9 @@
     <tool file="genomespace/genomespace_importer.xml" />
     <tool file="genomespace/genomespace_push.xml" />
   </section>
+  <section id="send" name="Send Data">
+    <tool file="data_export/export_remote.xml" />
+  </section>
   <section id="collection_operations" name="Collection Operations">
     <tool file="${model_tools_path}/unzip_collection.xml" />
     <tool file="${model_tools_path}/zip_collection.xml" />

diff --git a/galaxy/templates/_helpers.tpl b/galaxy/templates/_helpers.tpl
@@ -161,7 +161,7 @@ Creates the bash command for the handlers to wait for init scripts
 until [ -f /galaxy/server/config/mutable/db_init_done_{{$.Release.Revision}} ]; do echo "waiting for DB initialization"; sleep 1; done;
 until [ -f /galaxy/server/config/mutable/init_mounts_done_{{$.Release.Revision}} ]; do echo "waiting for copying onto NFS"; sleep 1; done;
 {{ if .Values.setupJob.downloadToolConfs.enabled }}
-until [ -f /galaxy/server/config/mutable/init_clone_done_{{$.Release.Revision}} ]; do echo "waiting for CVMFS copying"; sleep 1; done;
+until [ -f /galaxy/server/config/mutable/init_clone_done_{{$.Release.Revision}} ]; do echo "waiting for refdata copying"; sleep 1; done;
 {{- end }}
 {{- end -}}
 

diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml
@@ -1,4 +1,4 @@
-apiVersion: batch/v1beta1
+apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: {{ include "galaxy.fullname" . }}-maintenance
@@ -24,13 +24,13 @@ spec:
             command:
               - find
               - {{ .Values.persistence.mountPath }}/tmp
-              - '!' 
-              - -newermt 
+              - '!'
+              - -newermt
               - -{{ (index .Values "configs" "job_conf.yml" "runners" "k8s" "k8s_walltime_limit" | default 604800) }} seconds
-              - -type 
+              - -type
               - f
               - -exec
-              - rm 
+              - rm
               - '{}'
               - ;
             volumeMounts:

diff --git a/galaxy/templates/deployment-job.yaml b/galaxy/templates/deployment-job.yaml
@@ -142,9 +142,9 @@ spec:
               mountPath: {{ $.Values.setupJob.downloadToolConfs.volume.mountPath }}
               subPath: {{ $.Values.setupJob.downloadToolConfs.volume.subPath }}
             {{- end }}
-            {{- if $.Values.cvmfs.enabled }}
-            {{- range $key, $entry := $.Values.cvmfs.galaxyPersistentVolumeClaims }}
-            - name: cvmfs-gxy-{{ $key }}
+            {{- if $.Values.refdata.enabled }}
+            {{- range $key, $entry := $.Values.refdata.galaxyPersistentVolumeClaims }}
+            - name: refdata-gxy-{{ $key }}
               mountPath: {{ $entry.mountPath }}
             {{- end -}}
             {{- end }}
@@ -187,11 +187,11 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
-        {{- if $.Values.cvmfs.enabled }}
-        {{- range $key, $entry := $.Values.cvmfs.galaxyPersistentVolumeClaims }}
-        - name: cvmfs-gxy-{{ $key }}
+        {{- if $.Values.refdata.enabled }}
+        {{- range $key, $entry := $.Values.refdata.galaxyPersistentVolumeClaims }}
+        - name: refdata-gxy-{{ $key }}
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
         {{- end }}
         {{- end }}
         {{- if $.Values.extraVolumes }}

diff --git a/galaxy/templates/deployment-web.yaml b/galaxy/templates/deployment-web.yaml
@@ -133,9 +133,9 @@ spec:
               mountPath: {{ .Values.setupJob.downloadToolConfs.volume.mountPath }}
               subPath: {{ .Values.setupJob.downloadToolConfs.volume.subPath }}
             {{- end }}
-            {{- if .Values.cvmfs.enabled }}
-            {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-            - name: cvmfs-gxy-{{ $key }}
+            {{- if .Values.refdata.enabled }}
+            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+            - name: refdata-gxy-{{ $key }}
               mountPath: {{ $entry.mountPath }}
             {{- end -}}
             {{- end }}
@@ -181,11 +181,11 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
-        {{- if .Values.cvmfs.enabled }}
-        {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-        - name: cvmfs-gxy-{{ $key }}
+        {{- if .Values.refdata.enabled }}
+        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+        - name: refdata-gxy-{{ $key }}
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
         {{- end }}
         {{- end }}
         {{- if .Values.extraVolumes }}

diff --git a/galaxy/templates/deployment-workflow.yaml b/galaxy/templates/deployment-workflow.yaml
@@ -139,9 +139,9 @@ spec:
               mountPath: {{ .Values.setupJob.downloadToolConfs.volume.mountPath }}
               subPath: {{ .Values.setupJob.downloadToolConfs.volume.subPath }}
             {{- end }}
-            {{- if .Values.cvmfs.enabled }}
-            {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-            - name: cvmfs-gxy-{{ $key }}
+            {{- if .Values.refdata.enabled }}
+            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+            - name: refdata-gxy-{{ $key }}
               mountPath: {{ $entry.mountPath }}
             {{- end -}}
             {{- end }}
@@ -184,11 +184,11 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
-        {{- if .Values.cvmfs.enabled }}
-        {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-        - name: cvmfs-gxy-{{ $key }}
+        {{- if .Values.refdata.enabled }}
+        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+        - name: refdata-gxy-{{ $key }}
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
         {{- end }}
         {{- end }}
         {{- if .Values.extraVolumes }}

diff --git a/galaxy/templates/ingress-activity-canary.yaml b/galaxy/templates/ingress-activity-canary.yaml
@@ -6,14 +6,7 @@
 {{- $fullName := include "galaxy.fullname" . -}}
 {{- $ingressPath := .Values.ingress.path -}}
 {{- $servicePort := .Values.service.port -}}
-{{- $k8s_version := .Capabilities.KubeVersion.Version | toString }}
-{{- if semverCompare "^1.19.0-0" $k8s_version }}
 apiVersion: networking.k8s.io/v1
-{{- else if semverCompare "^1.14.0-0" $k8s_version -}}
-apiVersion: networking.k8s.io/v1beta1
-{{ else }}
-{{ fail "This chart requires Kubernetes v1.14 or later" }}
-{{- end }}
 kind: Ingress
 metadata:
   name: {{ $fullName }}-activity-canary
@@ -40,19 +33,13 @@ spec:
     - host: {{ .host | quote }}
       http:
         paths:
-          - path: {{ trimSuffix "/" $ingressPath }}/api/users/
-{{- if semverCompare "^1.19.0-0" $k8s_version }}
+          - path: {{ trimSuffix "/" $ingressPath }}/api/users
             pathType: Prefix
             backend:
               service:
                 name: {{ $fullName }}-nginx
                 port:
                   number: {{ $servicePort }}
-{{- else }}
-            backend:
-              serviceName: {{ $fullName }}-nginx
-              servicePort: {{ $servicePort }}
-{{- end }}
     {{- end }}
-  {{- end }}
+{{- end }}
 ---
diff --git a/galaxy/templates/ingress.yaml b/galaxy/templates/ingress.yaml
@@ -2,14 +2,7 @@
 {{- $fullName := include "galaxy.fullname" . -}}
 {{- $ingressPath := .Values.ingress.path -}}
 {{- $servicePort := .Values.service.port -}}
-{{- $k8s_version := .Capabilities.KubeVersion.Version | toString }}
-{{- if semverCompare "^1.19.0-0" $k8s_version -}}
 apiVersion: networking.k8s.io/v1
-{{- else if semverCompare "^1.14.0-0" $k8s_version -}}
-apiVersion: networking.k8s.io/v1beta1
-{{ else }}
-{{ fail "This chart requires Kubernetes v1.14 or later" }}
-{{- end }}
 kind: Ingress
 metadata:
   name: {{ $fullName }}
@@ -38,18 +31,12 @@ spec:
         paths:
           {{- range .paths }}
           - path: {{ .path }}
-{{- if semverCompare "^1.19.0-0" $k8s_version }}
             pathType: ImplementationSpecific
             backend:
               service:
                 name: {{ $fullName }}-nginx
                 port:
                   number: {{ $servicePort }}
-{{- else }}
-            backend:
-              serviceName: {{ $fullName }}-nginx
-              servicePort: {{ $servicePort }}
-{{- end }}
           {{- end }}
     {{- end }}
   {{- end }}

diff --git a/galaxy/templates/jobs-init.yaml b/galaxy/templates/jobs-init.yaml
@@ -26,7 +26,7 @@ spec:
       restartPolicy: OnFailure
       initContainers:
         - name: {{ .Chart.Name }}-init-postgres
-          image: alpine:3.7
+          image: alpine:3.16
           command: ['sh', '-c', 'chown 101:101 {{ .Values.persistence.mountPath }}; until nc -z -w3 {{ template "galaxy-postgresql.fullname" . }} 5432; do echo waiting for galaxy-postgres service; sleep 1; done;']
           volumeMounts: 
             - name: galaxy-data
@@ -54,7 +54,7 @@ spec:
           {{ include "galaxy.podEnvVars" . }}
           volumeMounts:
             {{- range $key, $entry := .Values.extraFileMappings -}}
-            {{- if $entry.applyToJob }}
+            {{- if $entry.applyToSetupJob }}
             - name: {{ include "galaxy.getExtraFilesUniqueName" $key }}
               mountPath: {{ $key }}
               subPath: {{ include "galaxy.getFilenameFromPath" $key }}
@@ -75,9 +75,9 @@ spec:
             - name: galaxy-data
               mountPath: /galaxy/server/config/mutable/
               subPath: config
-            {{- if .Values.cvmfs.enabled }}
-            {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-            - name: cvmfs-gxy-{{ $key }}
+            {{- if .Values.refdata.enabled }}
+            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+            - name: refdata-gxy-{{ $key }}
               mountPath: {{ $entry.mountPath }}
             {{- end -}}
             {{- end }}
@@ -115,11 +115,11 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
-        {{- if .Values.cvmfs.enabled }}
-        {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-        - name: cvmfs-gxy-{{ $key }}
+        {{- if .Values.refdata.enabled }}
+        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+        - name: refdata-gxy-{{ $key }}
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
         {{- end }}
         {{- end }}
         {{- if .Values.extraVolumes }}
@@ -182,9 +182,9 @@ spec:
             - name: galaxy-data
               mountPath: /galaxy/server/config/mutable/
               subPath: config
-            {{- if .Values.cvmfs.enabled }}
-            {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-            - name: cvmfs-gxy-{{ $key }}
+            {{- if .Values.refdata.enabled }}
+            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+            - name: refdata-gxy-{{ $key }}
               mountPath: {{ $entry.mountPath }}
             {{- end -}}
             {{- end }}
@@ -193,7 +193,7 @@ spec:
             {{- end }}
 {{ if .Values.setupJob.downloadToolConfs.enabled }}
         - name: {{ .Chart.Name }}-init-cloud-repo
-          image: alpine:3.7
+          image: alpine:3.16
           command: ['sh', '-c', '{{- include "galaxy.extract-archive-if-changed-command" (dict "extractPath" .Values.setupJob.downloadToolConfs.volume.mountPath "downloadUrl" .Values.setupJob.downloadToolConfs.archives.startup) -}} && echo "Done" > /galaxy/server/config/mutable/init_clone_done_{{.Release.Revision}}']
           volumeMounts:
             - name: galaxy-data
@@ -203,14 +203,14 @@ spec:
               mountPath: /galaxy/server/config/mutable/
               subPath: config
         - name: {{ .Chart.Name }}-init-cloud-repo-partial
-          image: alpine:3.7
+          image: alpine:3.16
           command: ['sh', '-c', '{{- include "galaxy.extract-archive-if-changed-command" (dict "extractPath" .Values.setupJob.downloadToolConfs.volume.mountPath "downloadUrl" .Values.setupJob.downloadToolConfs.archives.running) -}}']
           volumeMounts:
             - name: galaxy-data
               mountPath: {{ .Values.setupJob.downloadToolConfs.volume.mountPath }}
               subPath: {{ .Values.setupJob.downloadToolConfs.volume.subPath }}
         - name: {{ .Chart.Name }}-init-cloud-repo-full
-          image: alpine:3.7
+          image: alpine:3.16
           command: ['sh', '-c', '{{- include "galaxy.extract-archive-if-changed-command" (dict "extractPath" .Values.setupJob.downloadToolConfs.volume.mountPath "downloadUrl" .Values.setupJob.downloadToolConfs.archives.full) -}}']
           volumeMounts:
             - name: galaxy-data
@@ -248,11 +248,11 @@ spec:
           {{- else }}
           emptyDir: {}
           {{- end }}
-        {{- if .Values.cvmfs.enabled }}
-        {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims }}
-        - name: cvmfs-gxy-{{ $key }}
+        {{- if .Values.refdata.enabled }}
+        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
+        - name: refdata-gxy-{{ $key }}
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
         {{- end }}
         {{- end }}
         {{- if .Values.extraVolumes }}