From 3797ce434fa893e44e2b31607b02b95089720b10 Mon Sep 17 00:00:00 2001 From: bw2 Date: Sun, 17 Mar 2024 09:31:35 -0400 Subject: [PATCH] switched from gsutil cp to gcloud storage cp --- .idea/.gitignore | 2 ++ step_pipeline/batch.py | 22 +++++++++++----------- step_pipeline/pipeline.py | 5 ++++- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.idea/.gitignore b/.idea/.gitignore index 73f69e0..b0811f1 100644 --- a/.idea/.gitignore +++ b/.idea/.gitignore @@ -6,3 +6,5 @@ /dataSources.local.xml # Editor-based HTTP Client requests /httpRequests/ +# GitHub Copilot persisted chat sessions +/copilot/chatSessions diff --git a/step_pipeline/batch.py b/step_pipeline/batch.py index e9586bb..db8e060 100644 --- a/step_pipeline/batch.py +++ b/step_pipeline/batch.py @@ -788,7 +788,7 @@ def _preprocess_input_spec(self, input_spec): raise ValueError(f"Expected gs:// path but instead found '{input_spec.local_dir}'") self.gcloud_auth_activate_service_account() self.command(f"mkdir -p '{input_spec.local_dir}'") - self.command(self._generate_gsutil_copy_command( + self.command(self._generate_gcloud_copy_command( input_spec.original_source_path, output_dir=input_spec.local_dir)) self.command(f"ls -lh '{input_spec.local_path}'") # check that file was copied successfully @@ -817,7 +817,7 @@ def _preprocess_input_spec(self, input_spec): self._paths_localized_via_temp_bucket.add(temp_file_path) # copy file to temp bucket - gsutil_command = self._generate_gsutil_copy_command(source_path, output_dir=temp_dir) + gsutil_command = self._generate_gcloud_copy_command(source_path, output_dir=temp_dir) self.command(gsutil_command) # create an InputSpec with the updated source path @@ -885,7 +885,7 @@ def _get_size_of_all_inputs_localized_by_copy(self): return total_size_bytes - def _generate_gsutil_copy_command(self, source_path, output_dir=None, output_path=None, ignore_nonzero_exit_code=False): + def _generate_gcloud_copy_command(self, source_path, output_dir=None, output_path=None, ignore_nonzero_exit_code=False): """Utility method that puts together the gsutil command for copying the given source path to an output path or directory. Either the output path or the output directory must be provided. @@ -899,10 +899,10 @@ def _generate_gsutil_copy_command(self, source_path, output_dir=None, output_pat str: gsutil command string """ args = self._pipeline.parse_known_args() - gsutil_command = f"gsutil" + gcloud_copy_command = f"gcloud " if args.gcloud_project: - gsutil_command += f" -u {args.gcloud_project}" - + gcloud_copy_command += f"--project {args.gcloud_project} " + gcloud_copy_command += "storage cp " if output_path: destination = output_path elif output_dir: @@ -910,17 +910,17 @@ def _generate_gsutil_copy_command(self, source_path, output_dir=None, output_pat else: raise ValueError("Neither output_path nor output_dir arg was specified") - full_gsutil_command = f"time {gsutil_command} -m cp -r '{source_path}' '{destination}'" + full_gcloud_copy_command = f"time {gcloud_copy_command} --recursive '{source_path}' '{destination}'" if ignore_nonzero_exit_code: gsutil_command_with_error_handling = ( - f"({full_gsutil_command}) || (touch {os.path.basename(source_path)}{MARK_FILE_SUFFIX} && " - f"{gsutil_command} -m cp -r '{os.path.basename(source_path)}{MARK_FILE_SUFFIX}' '{destination}{MARK_FILE_SUFFIX}' " + f"({full_gcloud_copy_command}) || (touch {os.path.basename(source_path)}{MARK_FILE_SUFFIX} && " + f"{gcloud_copy_command} cp --recursive '{os.path.basename(source_path)}{MARK_FILE_SUFFIX}' '{destination}{MARK_FILE_SUFFIX}' " f") || true" ) return gsutil_command_with_error_handling else: - return full_gsutil_command + return full_gcloud_copy_command def _handle_input_transfer_using_cloudfuse(self, input_spec): """Utility method that implements localizing an input via cloudfuse. @@ -985,7 +985,7 @@ def _preprocess_output_spec(self, output_spec): raise ValueError(f"{output_spec.output_path} Destination path must start with gs://") self.gcloud_auth_activate_service_account() - self.command(self._generate_gsutil_copy_command( + self.command(self._generate_gcloud_copy_command( output_spec.local_path, output_path=output_spec.output_path, ignore_nonzero_exit_code=output_spec.optional)) diff --git a/step_pipeline/pipeline.py b/step_pipeline/pipeline.py index 8960d89..c66fe50 100644 --- a/step_pipeline/pipeline.py +++ b/step_pipeline/pipeline.py @@ -1075,7 +1075,10 @@ def switch_gcloud_auth_to_user_account(self, gcloud_credentials_path=None, gclou self.command(f"gcloud auth list") self.gcloud_auth_activate_service_account() - self.command(f"gsutil -m cp -r {os.path.join(gcloud_credentials_path, '.config')} /tmp/") + self.command("python3 -m pip install -U crcmod") + self.command(f"gcloud storage cp --recursive {os.path.join(gcloud_credentials_path, '.config')} /tmp/") + self.command("echo done copying .config") + self.command(f"rm -rf ~/.config") self.command(f"mv /tmp/.config ~/") self.command(f"gcloud config set account {gcloud_user_account}")