From 9c4bf884d286e77c752741372a0b5792c7060710 Mon Sep 17 00:00:00 2001
From: Dave McKay <d.mckay@epcc.ed.ac.uk>
Date: Tue, 29 Oct 2024 10:48:54 +0000
Subject: [PATCH] .

---
 csd3-side/scripts/backup_progress.py | 31 ++++++++++++++++++----------
 csd3-side/scripts/lsst-backup.py     |  4 ++--
 2 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/csd3-side/scripts/backup_progress.py b/csd3-side/scripts/backup_progress.py
index 680f50ca..88e633c8 100644
--- a/csd3-side/scripts/backup_progress.py
+++ b/csd3-side/scripts/backup_progress.py
@@ -1,25 +1,34 @@
 import tqdm
 import sys
 import string
+import os
 
 logp = sys.argv[1]
+logcsv = sys.argv[2]
 
 total = 0
 
 with open(logp, 'r') as logf:
     for line in logf:
-        if 'Folders: ' in line and '; Files: ' in line:
-            total = float(line.split()[3].strip().translate(str.maketrans('', '', string.punctuation)))
+        if 'Folders: ' in line and ' Files: ' in line:
+            total = float(line.split()[3].strip())
             break
 with tqdm.tqdm(total=total) as pbar:
     progress = 0
+    prev_len = 0
     while True:
-        with open(logp, 'r') as logf:
-            for line in logf:
-                if 'Folders: ' in line and '; Files: ' in line:
-                    prog = float(line.split()[1].strip().translate(str.maketrans('', '', string.punctuation)))
-                    if prog > progress:
-                        
-                        pbar.update(prog - progress)
-                        progress = prog
-                        break
+        with open(logcsv, 'r') as logc:
+            if len(logc.readlines()) > prev_len:
+                logc.seek(0)
+                prev_len = len(logc.readlines())
+                logc.seek(0)
+            else:
+                continue
+            for line in logc:
+                if 'collated_' in line:
+                    prog = len(line.split('"')[-2].split(','))
+                else:
+                    prog = 1
+                pbar.update(prog - progress)
+                progress = prog
+                
diff --git a/csd3-side/scripts/lsst-backup.py b/csd3-side/scripts/lsst-backup.py
index 3f562bbc..e30b1521 100644
--- a/csd3-side/scripts/lsst-backup.py
+++ b/csd3-side/scripts/lsst-backup.py
@@ -723,8 +723,8 @@ def process_files(s3_host, access_key, secret_key, bucket_name, current_objects,
     for folder, sub_folders, files in os.walk(local_dir, topdown=True):
         total_all_folders += 1
         total_all_files += len(files)
-        print(f'Folders: {total_all_folders}; Files: {total_all_files}.', flush=True, end='\r')
-    print()
+        print(f'F{"f"*total_all_files}', flush=True, end='')
+    print(f'Folders: {total_all_folders} Files: {total_all_files}', flush=True)
     if file_count_stop:
         total_non_collate_zip = len(current_objects[current_objects['CURRENT_OBJECTS'].str.contains('collated_') == False])
         if total_non_collate_zip == total_all_files: