From b0a09b506970ec81bef217b18064f600c5f8fabb Mon Sep 17 00:00:00 2001 From: bw2 Date: Tue, 22 Oct 2024 19:27:23 -0400 Subject: [PATCH] fixed bucket ls for requester-pays buckets --- step_pipeline/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/step_pipeline/utils.py b/step_pipeline/utils.py index 63c3e93..4c3747f 100644 --- a/step_pipeline/utils.py +++ b/step_pipeline/utils.py @@ -417,8 +417,9 @@ def check_gcloud_storage_region(gs_path, expected_regions=("US", "US-CENTRAL1"), #client = _get_google_storage_client(gcloud_project=gcloud_project) #bucket = client.get_bucket(bucket_name) #location = bucket.location + gcloud_project_arg = f"-u {gcloud_project}" if gcloud_project else "" gsutil_output = subprocess.check_output( - f"gsutil ls -Lb gs://{bucket_name}", + f"gsutil {gcloud_project_arg} ls -Lb gs://{bucket_name}", shell=True, stderr=subprocess.STDOUT, encoding="UTF-8") @@ -431,7 +432,7 @@ def check_gcloud_storage_region(gs_path, expected_regions=("US", "US-CENTRAL1"), break if location is None: raise GoogleStorageException(f"ERROR: Could not determine gs://{bucket_name} bucket region." - f"gsutil ls -Lb gs://{bucket_name} returned:\n{gsutil_output}") + f"gsutil {gcloud_project_arg} ls -Lb gs://{bucket_name} returned:\n{gsutil_output}") BUCKET_LOCATION_CACHE[bucket_name] = location except Exception as e: