From fa7ecf9a3c1cc9eed67ec2c33711bf9873e1f9c2 Mon Sep 17 00:00:00 2001 From: bw2 Date: Fri, 26 Apr 2024 10:31:02 -0400 Subject: [PATCH] fixed localization from requester-pays buckets --- step_pipeline/batch.py | 3 +++ step_pipeline/utils.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/step_pipeline/batch.py b/step_pipeline/batch.py index db8e060..62fe0d0 100644 --- a/step_pipeline/batch.py +++ b/step_pipeline/batch.py @@ -903,6 +903,9 @@ def _generate_gcloud_copy_command(self, source_path, output_dir=None, output_pat if args.gcloud_project: gcloud_copy_command += f"--project {args.gcloud_project} " gcloud_copy_command += "storage cp " + if args.gcloud_project: + gcloud_copy_command += f"--billing-project {args.gcloud_project} " + if output_path: destination = output_path elif output_dir: diff --git a/step_pipeline/utils.py b/step_pipeline/utils.py index 84d159b..8607aa5 100644 --- a/step_pipeline/utils.py +++ b/step_pipeline/utils.py @@ -20,7 +20,7 @@ GSUTIL_PATH_TO_FILE_STAT_CACHE = {} BUCKET_LOCATION_CACHE = {} _BUCKET_ERROR_COUNTER = collections.defaultdict(int) -_MAX_ERROR_MESSAGES_TO_PRINT_PER_BUCKET = 1000 +_MAX_ERROR_MESSAGES_TO_PRINT_PER_BUCKET = 5 LOCAL_TIMEZONE = pytz.timezone("US/Eastern") #datetime.now(timezone.utc).astimezone().tzinfo