fixed localization from requester-pays buckets

bw2 · Apr 26, 2024 · fa7ecf9 · fa7ecf9
1 parent 40d7693
commit fa7ecf9
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/step_pipeline/batch.py b/step_pipeline/batch.py
@@ -903,6 +903,9 @@ def _generate_gcloud_copy_command(self, source_path, output_dir=None, output_pat
         if args.gcloud_project:
             gcloud_copy_command += f"--project {args.gcloud_project} "
         gcloud_copy_command += "storage cp "
+        if args.gcloud_project:
+            gcloud_copy_command += f"--billing-project {args.gcloud_project} "
+
         if output_path:
             destination = output_path
         elif output_dir:

diff --git a/step_pipeline/utils.py b/step_pipeline/utils.py
@@ -20,7 +20,7 @@
 GSUTIL_PATH_TO_FILE_STAT_CACHE = {}
 BUCKET_LOCATION_CACHE = {}
 _BUCKET_ERROR_COUNTER = collections.defaultdict(int)
-_MAX_ERROR_MESSAGES_TO_PRINT_PER_BUCKET = 1000
+_MAX_ERROR_MESSAGES_TO_PRINT_PER_BUCKET = 5
 
 LOCAL_TIMEZONE = pytz.timezone("US/Eastern") #datetime.now(timezone.utc).astimezone().tzinfo