From 231475317ec666f8acb46d00a044726091cb29ab Mon Sep 17 00:00:00 2001 From: Dave McKay Date: Fri, 20 Dec 2024 09:49:30 +0000 Subject: [PATCH] add script to loop backup --- csd3-side/scripts/loop_backup.sh | 10 ++++++++++ csd3-side/scripts/lsst-backup.py | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 csd3-side/scripts/loop_backup.sh diff --git a/csd3-side/scripts/loop_backup.sh b/csd3-side/scripts/loop_backup.sh new file mode 100644 index 0000000..aeac0e2 --- /dev/null +++ b/csd3-side/scripts/loop_backup.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Loop backup script +# This script will run the backup script until all the files are backed up +# It's kind of brute force. +config_file=$1 +collate_list_file=$2 +while [ $(grep -c True $collate_list_file) -gt 0 ] +do + python ../../scripts/lsst-backup.py --config-file $config_file +done \ No newline at end of file diff --git a/csd3-side/scripts/lsst-backup.py b/csd3-side/scripts/lsst-backup.py index 3052cdd..4660653 100644 --- a/csd3-side/scripts/lsst-backup.py +++ b/csd3-side/scripts/lsst-backup.py @@ -1177,7 +1177,6 @@ def process_files(s3, bucket_name, api, current_objects, exclude, local_dir, des for id in to_collate_uploads['id']: if len(upload_futures) >= len(client.scheduler_info()['workers'])*2: while len(upload_futures) >= len(client.scheduler_info()['workers'])*2: - print(len(upload_futures), flush=True) for ulf in upload_futures: if 'exception' in ulf.status or 'error' in ulf.status: f_tuple = ulf.exception(), ulf.traceback()