86 bug premature closure of dask client (#92)

* clean (#87) (#88) * test put_object response with infinite loop * response dicts passed * exit after printing dicts * typos * print response status * clarify output * existence of a response may be enough * use status 201 * add script to loop backup * make loop_backup.sh executable * . * . * rb (#91) * clean (#87) * 86 bug premature closure of dask client (#90) * clean (#87) (#88) * test put_object response with infinite loop * response dicts passed * exit after printing dicts * typos * print response status * clarify output * existence of a response may be enough * use status 201 * add script to loop backup * make loop_backup.sh executable * .
lsst-uk · Dec 20, 2024 · e290370 · e290370
1 parent 3a602e8
commit e290370
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 14 deletions.
diff --git a/echo-side/containers/mamba_env/Dockerfile b/echo-side/containers/mamba_env/Dockerfile
@@ -1,11 +1,9 @@
-FROM continuumio/miniconda3
+FROM condaforge/miniforge3
 
 LABEL org.opencontainers.image.source https://github.com/lsst-uk/csd3-echo-somerville
 LABEL org.opencontainers.image.description="Code to backup and curate LSST-UK data from CSD3 to Echo."
 LABEL org.opencontainers.image.licenses="Apache-2.0"
 
-RUN conda install -y -c conda-forge mamba && mamba init
-
 COPY environment.yaml /environment.yaml
 
 RUN mamba env create --name lsst-uk --file=/environment.yaml

diff --git a/scripts/process_collated_zips.py b/scripts/process_collated_zips.py
@@ -154,7 +154,7 @@ def extract_and_upload_mp(bucket_name, debug, zipfile_key):
                 pbar.update(zf.getinfo(content_file).file_size)
 
 def extract_and_upload_zipfiles(extract_list, bucket_name, pool_size, debug):
-    print(f'Extracting zip files and uploading contents using {pool_size} processes...') 
+    print(f'Extracting zip files and uploading contents using {pool_size} processes...')
     with Pool(pool_size) as p:
         p.map(partial(extract_and_upload_mp, bucket_name, debug), extract_list)#, chunksize=len(extract_list)//pool_size)
 
@@ -227,12 +227,12 @@ def error(self, message):
         debug = True
     else:
         debug = False
-    
+
     if args.extract:
         extract = True
     else:
         extract = False
-    
+
     if args.nprocs:
         nprocs = args.nprocs
         if nprocs < 1:
@@ -247,13 +247,12 @@ def error(self, message):
         get_contents_metadata = True
 
     # Setup bucket object
-
     try:
         assert bm.check_keys()
     except AssertionError as e:
         print(e)
         sys.exit()
-    
+
     s3 = bm.get_resource()
     bucket_list = bm.bucket_list(s3)
 
@@ -266,7 +265,7 @@ def error(self, message):
     if list_contents:
         for i in range(len(zipfiles_df)):
             print(f'{zipfiles_df.iloc[i]["zipfile"]}: {zipfiles_df.iloc[i]["contents"]}')
-    
+
     if verify_contents:
         print('Verifying zip file contents...')
         zipfiles_df = prepend_zipfile_path_to_contents(zipfiles_df, debug)
@@ -276,9 +275,7 @@ def error(self, message):
             print(extract_list)
         else:
             print('All zip files previously extracted.')
-        # for zipfile in extract_list:
-        #     print(zipfile)
-
+
     if extract:
         print('Extracting zip files...')
         zipfiles_df = prepend_zipfile_path_to_contents(zipfiles_df, debug)
@@ -291,9 +288,7 @@ def error(self, message):
         else:
             print('All zip files previously extracted.')
 
-
     print('Done.')
-
 
 if __name__ == '__main__':
     main()