.

csd3-side/scripts/backup_progress.py

@@ -1,25 +1,34 @@
 import tqdm
 import sys
 import string
+import os
 
 logp = sys.argv[1]
+logcsv = sys.argv[2]
 
 total = 0
 
 with open(logp, 'r') as logf:
     for line in logf:
-        if 'Folders: ' in line and '; Files: ' in line:
-            total = float(line.split()[3].strip().translate(str.maketrans('', '', string.punctuation)))
+        if 'Folders: ' in line and ' Files: ' in line:
+            total = float(line.split()[3].strip())
             break
 with tqdm.tqdm(total=total) as pbar:
     progress = 0
+    prev_len = 0
     while True:
-        with open(logp, 'r') as logf:
-            for line in logf:
-                if 'Folders: ' in line and '; Files: ' in line:
-                    prog = float(line.split()[1].strip().translate(str.maketrans('', '', string.punctuation)))
-                    if prog > progress:
-
-                        pbar.update(prog - progress)
-                        progress = prog
-                        break
+        with open(logcsv, 'r') as logc:
+            if len(logc.readlines()) > prev_len:
+                logc.seek(0)
+                prev_len = len(logc.readlines())
+                logc.seek(0)
+            else:
+                continue
+            for line in logc:
+                if 'collated_' in line:
+                    prog = len(line.split('"')[-2].split(','))
+                else:
+                    prog = 1
+                pbar.update(prog - progress)
+                progress = prog
+

csd3-side/scripts/lsst-backup.py

@@ -723,8 +723,8 @@ def process_files(s3_host, access_key, secret_key, bucket_name, current_objects,
     for folder, sub_folders, files in os.walk(local_dir, topdown=True):
         total_all_folders += 1
         total_all_files += len(files)
-        print(f'Folders: {total_all_folders}; Files: {total_all_files}.', flush=True, end='\r')
-    print()
+        print(f'F{"f"*total_all_files}', flush=True, end='')
+    print(f'Folders: {total_all_folders} Files: {total_all_files}', flush=True)
     if file_count_stop:
         total_non_collate_zip = len(current_objects[current_objects['CURRENT_OBJECTS'].str.contains('collated_') == False])
         if total_non_collate_zip == total_all_files: