fix: apply symlink replacement

Signed-off-by: chirichidi <[email protected]>
airflow-helm · Feb 19, 2024 · a94678a · a94678a
1 parent e0a2244
commit a94678a
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 3 deletions.
diff --git a/charts/airflow/templates/_helpers/pods.tpl b/charts/airflow/templates/_helpers/pods.tpl
@@ -324,11 +324,29 @@ EXAMPLE USAGE: {{ include "airflow.container.s3_sync" (dict "Release" .Release "
     - "-c"
     {{- if $.sync_one_time }}
     - |
-      aws s3 cp --recursive s3://{{ .Values.dags.s3Sync.bucket }}/{{ .Values.dags.s3Sync.s3Path }} {{ include "airflow.dags.path" . }}
+      BASE_DIR={{ .Values.dags.path | trimSuffix "/" }}
+      TEMP_DIR=$(mktemp -u $BASE_DIR/tmp.XXXXXX)
+      SYM_LINK="{{ include "airflow.dags.path" . | trimSuffix "/" }}"
+
+      aws s3 cp --recursive s3://{{ .Values.dags.s3Sync.bucket }}/{{ .Values.dags.s3Sync.s3Path }} $TEMP_DIR \
+        --exclude "*.pyc"
+      ln -s $TEMP_DIR $SYM_LINK
     {{- else }}
     - |
+      BASE_DIR="{{ .Values.dags.path | trimSuffix "/" }}"
+      SYM_LINK="{{ include "airflow.dags.path" . | trimSuffix "/" }}"
       while true; do
-          aws s3 sync --delete s3://{{ .Values.dags.s3Sync.bucket }}/{{ .Values.dags.s3Sync.s3Path }} {{ include "airflow.dags.path" . }}
+          TEMP_DIR=$(mktemp -u $BASE_DIR/tmp.XXXXXX)
+          CURR_DIR=$(readlink -f "$SYM_LINK")
+
+          cp -a $CURR_DIR $TEMP_DIR
+          aws s3 sync --delete s3://{{ .Values.dags.s3Sync.bucket }}/{{ .Values.dags.s3Sync.s3Path }} $TEMP_DIR \
+            --exclude "*.pyc"
+          if [ $? -eq 0 ]; then
+            rm -f $SYM_LINK
+            ln -s $TEMP_DIR $SYM_LINK
+            rm -rf $CURR_DIR
+          fi
           sleep {{ $.Values.dags.s3Sync.syncWait }}
       done
     {{- end }}

diff --git a/charts/airflow/values.yaml b/charts/airflow/values.yaml
@@ -1464,7 +1464,7 @@ dags:
   ## configs for a sync from s3 object storage
   ##
   s3Sync:
-    ## if the git-sync sidecar container is enabled
+    ## if the s3-sync sidecar container is enabled
     ##
     enabled: false