add script to loop backup

lsst-uk · Dec 20, 2024 · 2314753 · 2314753
1 parent b942507
commit 2314753
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 1 deletion.
diff --git a/csd3-side/scripts/loop_backup.sh b/csd3-side/scripts/loop_backup.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+# Loop backup script
+# This script will run the backup script until all the files are backed up
+# It's kind of brute force.
+config_file=$1
+collate_list_file=$2
+while [ $(grep -c True $collate_list_file) -gt 0 ]
+do
+    python ../../scripts/lsst-backup.py --config-file $config_file
+done
diff --git a/csd3-side/scripts/lsst-backup.py b/csd3-side/scripts/lsst-backup.py
@@ -1177,7 +1177,6 @@ def process_files(s3, bucket_name, api, current_objects, exclude, local_dir, des
         for id in to_collate_uploads['id']:
             if len(upload_futures) >= len(client.scheduler_info()['workers'])*2:
                 while len(upload_futures) >= len(client.scheduler_info()['workers'])*2:
-                    print(len(upload_futures), flush=True)
                     for ulf in upload_futures:
                         if 'exception' in ulf.status or 'error' in ulf.status:
                             f_tuple = ulf.exception(), ulf.traceback()