diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63844271b..6b85135b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,7 @@ env: jobs: lint: + timeout-minutes: 60 runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -72,6 +73,7 @@ jobs: if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead run: nox -vs cleanup_buckets test: + timeout-minutes: 90 needs: cleanup_buckets env: B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} @@ -131,6 +133,7 @@ jobs: if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' && contains(fromJSON('["3.7", "pypy3.10", "3.12"]'), matrix.python-version) }} run: nox -vs integration -p ${{ matrix.python-version }} -- -m "require_secrets" --cleanup test-docker: + timeout-minutes: 90 needs: cleanup_buckets env: B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} @@ -163,6 +166,7 @@ jobs: if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} run: nox -vs docker_test -- backblazeit/b2:test test-linux-bundle: + timeout-minutes: 90 needs: cleanup_buckets env: B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} @@ -201,6 +205,7 @@ jobs: if-no-files-found: warn retention-days: 7 test-windows-bundle: + timeout-minutes: 90 needs: cleanup_buckets env: B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} @@ -240,6 +245,7 @@ jobs: if-no-files-found: warn retention-days: 7 doc: + timeout-minutes: 30 needs: build runs-on: ubuntu-latest steps: diff --git a/b2/_internal/_cli/b2args.py b/b2/_internal/_cli/b2args.py index ba744f6f3..86cfe6fe8 100644 --- a/b2/_internal/_cli/b2args.py +++ b/b2/_internal/_cli/b2args.py @@ -78,6 +78,14 @@ def parse_bucket_name(value: str, allow_all_buckets: bool = False) -> str: return str(value) +def b2id_or_file_like_b2_uri_or_bucket_name(value: str) -> Union[B2URIBase, str]: + try: + bucket_name = parse_bucket_name(value) + return bucket_name + except ValueError: + return b2id_or_file_like_b2_uri(value) + + B2ID_URI_ARG_TYPE = wrap_with_argument_type_error(b2id_uri) B2_BUCKET_URI_ARG_TYPE = wrap_with_argument_type_error(b2_bucket_uri) B2ID_OR_B2_URI_ARG_TYPE = wrap_with_argument_type_error(parse_b2_uri) @@ -86,6 +94,9 @@ def parse_bucket_name(value: str, allow_all_buckets: bool = False) -> str: functools.partial(parse_b2_uri, allow_all_buckets=True) ) B2ID_OR_FILE_LIKE_B2_URI_ARG_TYPE = wrap_with_argument_type_error(b2id_or_file_like_b2_uri) +B2ID_OR_FILE_LIKE_B2_URI_OR_BUCKET_NAME_ARG_TYPE = wrap_with_argument_type_error( + b2id_or_file_like_b2_uri_or_bucket_name +) def add_bucket_name_argument( @@ -194,5 +205,18 @@ def add_b2id_or_file_like_b2_uri_argument(parser: argparse.ArgumentParser, name= ).completer = b2uri_file_completer +def add_b2id_or_file_like_b2_uri_or_bucket_name_argument( + parser: argparse.ArgumentParser, name="B2_URI" +): + """ + Add a B2 URI pointing to a file as an argument to the parser. + """ + parser.add_argument( + name, + type=B2ID_OR_FILE_LIKE_B2_URI_OR_BUCKET_NAME_ARG_TYPE, + help="B2 URI pointing to a file, e.g. b2://yourBucket/file.txt or b2id://fileId", + ).completer = b2uri_file_completer + + def get_keyid_and_key_from_env_vars() -> Tuple[Optional[str], Optional[str]]: return environ.get(B2_APPLICATION_KEY_ID_ENV_VAR), environ.get(B2_APPLICATION_KEY_ENV_VAR) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 15c9177fe..662443d2b 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -140,6 +140,7 @@ add_b2id_or_b2_bucket_uri_argument, add_b2id_or_b2_uri_argument, add_b2id_or_file_like_b2_uri_argument, + add_b2id_or_file_like_b2_uri_or_bucket_name_argument, add_b2id_uri_argument, add_bucket_name_argument, get_keyid_and_key_from_env_vars, @@ -712,6 +713,21 @@ def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase: return args.B2_URI +class B2URIFileOrBucketNameFileNameArgMixin: + @classmethod + def _setup_parser(cls, parser): + add_b2id_or_file_like_b2_uri_or_bucket_name_argument(parser) + parser.add_argument('fileName', nargs='?', help=argparse.SUPPRESS) + super()._setup_parser(parser) + + def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase | str: + if isinstance(args.B2_URI, B2URI): + return args.B2_URI + + bucket_name = args.B2_URI + return bucket_name + + class B2URIFileIDArgMixin: @classmethod def _setup_parser(cls, parser): @@ -2161,6 +2177,22 @@ class FileHideBase(Command): - **writeFiles** """ + def _run(self, args): + b2_uri = self.get_b2_uri_from_arg(args) + if isinstance(b2_uri, B2URI): + bucket_name = b2_uri.bucket_name + file_name = b2_uri.path + else: + bucket_name = b2_uri + file_name = args.fileName + + bucket = self.api.get_bucket_by_name(bucket_name) + file_info = bucket.hide_file(file_name) + self._print_json(file_info) + return 0 + + +class HideFileBase(Command): @classmethod def _setup_parser(cls, parser): add_bucket_name_argument(parser) @@ -2174,6 +2206,33 @@ def _run(self, args): return 0 +class FileUnhideBase(Command): + """ + Delete the "hide marker" for a given file. + + Requires capability: + + - **listFiles** + - **deleteFiles** + + and optionally: + + - **bypassGovernance** + """ + + @classmethod + def _setup_parser(cls, parser): + add_normalized_argument(parser, '--bypass-governance', action='store_true', default=False) + super()._setup_parser(parser) + + def _run(self, args): + b2_uri = self.get_b2_uri_from_arg(args) + bucket = self.api.get_bucket_by_name(b2_uri.bucket_name) + file_id_and_name = bucket.unhide_file(b2_uri.path, args.bypass_governance) + self._print_json(file_id_and_name) + return 0 + + class BucketListBase(Command): """ List all of the buckets in the current account. @@ -5073,11 +5132,17 @@ class FileCopyById(FileCopyByIdBase): @File.subcommands_registry.register -class FileHide(FileHideBase): +class FileHide(B2URIFileOrBucketNameFileNameArgMixin, FileHideBase): __doc__ = FileHideBase.__doc__ COMMAND_NAME = 'hide' +@File.subcommands_registry.register +class FileUnhide(B2URIFileArgMixin, FileUnhideBase): + __doc__ = FileUnhideBase.__doc__ + COMMAND_NAME = 'unhide' + + @File.subcommands_registry.register class FileUpdate(FileUpdateBase): __doc__ = FileUpdateBase.__doc__ @@ -5145,7 +5210,7 @@ class CopyFileById(CmdReplacedByMixin, FileCopyByIdBase): replaced_by_cmd = (File, FileCopyById) -class HideFile(CmdReplacedByMixin, FileHideBase): +class HideFile(CmdReplacedByMixin, HideFileBase): __doc__ = FileHideBase.__doc__ replaced_by_cmd = (File, FileHide) diff --git a/changelog.d/+b2_file_hide.added.md b/changelog.d/+b2_file_hide.added.md new file mode 100644 index 000000000..df657d7f9 --- /dev/null +++ b/changelog.d/+b2_file_hide.added.md @@ -0,0 +1 @@ +Support new `b2_uri` and deprecated `bucket_name file_name` arguments in `b2 file hide`. diff --git a/changelog.d/+b2_file_unhide.added.md b/changelog.d/+b2_file_unhide.added.md new file mode 100644 index 000000000..3e4c7612c --- /dev/null +++ b/changelog.d/+b2_file_unhide.added.md @@ -0,0 +1 @@ +Add `b2 file unhide` command. \ No newline at end of file diff --git a/changelog.d/554.fixed.md b/changelog.d/554.fixed.md new file mode 100644 index 000000000..c6967573b --- /dev/null +++ b/changelog.d/554.fixed.md @@ -0,0 +1 @@ +Update `b2sdk` to 2.5.0, to fix `TruncatedOutput` download errors when network is congested (e.g., due use of high downloader thread count). diff --git a/pdm.lock b/pdm.lock index 5213fd7cb..29913ff84 100644 --- a/pdm.lock +++ b/pdm.lock @@ -4,8 +4,8 @@ [metadata] groups = ["default", "bundle", "doc", "format", "full", "license", "lint", "release", "test"] strategy = ["cross_platform", "inherit_metadata"] -lock_version = "4.4.1" -content_hash = "sha256:5d1f8819c3c22b05f569213b507fa9c43bc13b67e6b2006478541b32945eefc9" +lock_version = "4.4.2" +content_hash = "sha256:c618124f32be53fa248fce6237836afabf3bfe54ac71474f16d509276070b541" [[package]] name = "alabaster" @@ -100,7 +100,7 @@ files = [ [[package]] name = "b2sdk" -version = "2.4.1" +version = "2.5.0" requires_python = ">=3.7" summary = "Backblaze B2 SDK" groups = ["default"] @@ -112,8 +112,8 @@ dependencies = [ "typing-extensions>=4.7.1; python_version < \"3.12\"", ] files = [ - {file = "b2sdk-2.4.1-py3-none-any.whl", hash = "sha256:7ce6301cb119fdbf9826d4a11c7ea783194a2ffbfaf2fb1c437c3643e22bfe36"}, - {file = "b2sdk-2.4.1.tar.gz", hash = "sha256:257510b7c661dc6d513588560b436f2424fcbb4213834fa3ca52c7a5fd8d44b7"}, + {file = "b2sdk-2.5.0-py3-none-any.whl", hash = "sha256:38bf80a6cd2bff5b104f7fd10023cf9e338c6f9d41da6108db5ad68f76f5abaa"}, + {file = "b2sdk-2.5.0.tar.gz", hash = "sha256:d7c20125e64508a730e56307d75284790079cdb88e63851fff820a09b24fb1d9"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index c6c6c18bc..c6136c654 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ dependencies = [ "argcomplete>=2,<4", "arrow>=1.0.2,<2.0.0", - "b2sdk>=2.4.1,<3", + "b2sdk>=2.5.0,<3", "docutils>=0.18.1", "idna~=3.4; platform_system == 'Java'", "importlib-metadata>=3.3; python_version < '3.8'", diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index 242d2485c..f799fdfef 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -357,6 +357,20 @@ def test_basic(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args, apiver_ ) should_equal(['a', 'b/1', 'b/2', 'd'], [f['fileName'] for f in list_of_files]) + b2_tool.should_succeed(['file', 'unhide', f'b2://{bucket_name}/c']) + + list_of_files = b2_tool.should_succeed_json( + ['ls', '--json', '--recursive', *b2_uri_args(bucket_name)] + ) + should_equal(['a', 'b/1', 'b/2', 'c', 'd'], [f['fileName'] for f in list_of_files]) + + b2_tool.should_succeed(['file', 'hide', bucket_name, 'c']) + + list_of_files = b2_tool.should_succeed_json( + ['ls', '--json', '--recursive', *b2_uri_args(bucket_name)] + ) + should_equal(['a', 'b/1', 'b/2', 'd'], [f['fileName'] for f in list_of_files]) + list_of_files = b2_tool.should_succeed_json( ['ls', '--json', '--recursive', '--versions', *b2_uri_args(bucket_name)] ) diff --git a/test/unit/_cli/test_autocomplete_cache.py b/test/unit/_cli/test_autocomplete_cache.py index 383e224b3..41e0c7364 100644 --- a/test/unit/_cli/test_autocomplete_cache.py +++ b/test/unit/_cli/test_autocomplete_cache.py @@ -176,7 +176,7 @@ def test_complete_with_escaped_control_characters( store=autocomplete_cache.HomeCachePickleStore(tmp_path), ) - with autocomplete_runner(f'b2 file hide {bucket} '): + with autocomplete_runner(f'b2 hide-file {bucket} '): exit, argcomplete_output = argcomplete_result() assert exit == 0 assert escaped_cc_file_name in argcomplete_output @@ -200,7 +200,7 @@ def test_complete_with_file_suggestions( tracker=autocomplete_cache.VersionTracker(), store=autocomplete_cache.HomeCachePickleStore(tmp_path), ) - with autocomplete_runner(f'b2 file hide {bucket} '): + with autocomplete_runner(f'b2 hide-file {bucket} '): exit, argcomplete_output = argcomplete_result() assert exit == 0 assert file_name in argcomplete_output diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index edec02f1a..5335cc8fe 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -1025,6 +1025,72 @@ def test_rm_fileid_v4(self): # Delete one file version self._run_command(['rm', 'b2id://9999']) + def test_hide_file_legacy_syntax(self): + self._authorize_account() + self._run_command(['bucket', 'create', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0) + + with TempDir() as temp_dir: + local_file1 = self._make_local_file(temp_dir, 'file1.txt') + # For this test, use a mod time without millis. My mac truncates + # millis and just leaves seconds. + mod_time = 1500111222 + os.utime(local_file1, (mod_time, mod_time)) + self.assertEqual(1500111222, os.path.getmtime(local_file1)) + + # Upload a file + self._run_command( + [ + 'file', 'upload', '--no-progress', 'my-bucket', local_file1, 'file1.txt', + '--cache-control=private, max-age=3600' + ], + remove_version=True, + ) + + # Get file info + expected_json = { + "accountId": self.account_id, + "action": "upload", + "bucketId": "bucket_0", + "size": 11, + "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", + "contentType": "b2/x-auto", + "fileId": "9999", + "fileInfo": + { + "src_last_modified_millis": "1500111222000", + "b2-cache-control": "private, max-age=3600" + }, + "fileName": "file1.txt", + "serverSideEncryption": { + "mode": "none" + }, + "uploadTimestamp": 5000 + } + + self._run_command( + ['file', 'info', 'b2id://9999'], + expected_json_in_stdout=expected_json, + ) + + # Hide the file + expected_json = { + "action": "hide", + "contentSha1": "none", + "fileId": "9998", + "fileInfo": {}, + "fileName": "file1.txt", + "serverSideEncryption": { + "mode": "none" + }, + "size": 0, + "uploadTimestamp": 5001 + } + + self._run_command( + ['file', 'hide', 'my-bucket', 'file1.txt'], + expected_json_in_stdout=expected_json, + ) + def test_files(self): self._authorize_account() @@ -2077,11 +2143,14 @@ def test_get_bucket_with_hidden(self): # something has failed if the output of 'bucket get' does not match the canon. stdout, stderr = self._get_stdouterr() console_tool = self.console_tool_class(stdout, stderr) - console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', 'hidden1']) + console_tool.run_command(['b2', 'file', 'hide', 'b2://my-bucket/hidden1']) console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', 'hidden2']) console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', 'hidden3']) console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden4']) + # unhide one file + console_tool.run_command(['b2', 'file', 'unhide', 'b2://my-bucket/hidden2']) + # Now check the output of `bucket get` against the canon. expected_json = { "accountId": self.account_id, @@ -2093,7 +2162,7 @@ def test_get_bucket_with_hidden(self): "defaultServerSideEncryption": { "mode": "none" }, - "fileCount": 10, + "fileCount": 9, "lifecycleRules": [], "options": [], "revision": 1, @@ -2138,14 +2207,18 @@ def test_get_bucket_complex(self): # something has failed if the output of 'bucket get' does not match the canon. stdout, stderr = self._get_stdouterr() console_tool = self.console_tool_class(stdout, stderr) + console_tool.run_command(['b2', 'file', 'hide', 'b2://my-bucket/1/hidden1']) console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/hidden1']) - console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/hidden1']) - console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/hidden2']) + console_tool.run_command(['b2', 'file', 'hide', 'b2://my-bucket/1/hidden2']) console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/2/hidden3']) console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/2/hidden3']) console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/2/hidden3']) console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/2/hidden3']) + # Unhide a file + console_tool.run_command(['b2', 'file', 'unhide', 'b2://my-bucket/1/hidden2']) + console_tool.run_command(['b2', 'file', 'unhide', 'b2://my-bucket/1/hidden2']) + # Now check the output of `bucket get` against the canon. expected_json = { "accountId": self.account_id, @@ -2157,7 +2230,7 @@ def test_get_bucket_complex(self): "defaultServerSideEncryption": { "mode": "none" }, - "fileCount": 29, + "fileCount": 28, "lifecycleRules": [], "options": [], "revision": 1,