diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d7e6429a..6e6613b4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +* Add `cat` command for downloading file contents directly to stdout + ### Fixed * Emit `Using https://api.backblazeb2.com` message to stderr instead of stdout, therefor prevent JSON output corruption ### Changed * Stream `ls --json` JSON output instead of dumping it only after all objects have been fetched +* Alias `-` to stdout in `download-file-by-name` or `download-file-by-id` command ## [3.12.0] - 2023-10-28 @@ -58,7 +62,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Add s3 endpoint to `get-account-info` ### Deprecated -* Support of `-` as a valid filename in `upload-file` command. In future `-` will be an alias for standard input. +* Deprecate support of `-` as a valid filename in `upload-file` command. In the future `-` will always be interpreted as standard input ### Changed * Better help text for --corsRules diff --git a/b2/_utils/filesystem.py b/b2/_utils/filesystem.py index 8e529bda6..cb3d7fa5b 100644 --- a/b2/_utils/filesystem.py +++ b/b2/_utils/filesystem.py @@ -7,6 +7,7 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +import platform import stat from pathlib import Path @@ -18,3 +19,6 @@ def points_to_fifo(path: Path) -> bool: return stat.S_ISFIFO(path.stat().st_mode) except OSError: return False + + +STDOUT_FILE_PATH = "CON" if platform.system() == "Windows" else "/dev/stdout" diff --git a/b2/_utils/uri.py b/b2/_utils/uri.py new file mode 100644 index 000000000..bd282bacf --- /dev/null +++ b/b2/_utils/uri.py @@ -0,0 +1,76 @@ +###################################################################### +# +# File: b2/_utils/uri.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from __future__ import annotations + +import dataclasses +import pathlib +import urllib +from pathlib import Path + + +class B2URIBase: + pass + + +@dataclasses.dataclass +class B2URI(B2URIBase): + bucket: str + path: str + + def __str__(self) -> str: + return f"b2://{self.bucket}{self.path}" + + def is_dir(self) -> bool: + """ + Return if the path is a directory. + + Please note this is symbolical. + It is possible for file to have a trailing slash, but it is HIGHLY discouraged, and not supported by B2 CLI. + + :return: True if the path is a file, False if it's a directory + """ + return self.path.endswith("/") + + +@dataclasses.dataclass +class B2FileIdURI(B2URIBase): + file_id: str + + def __str__(self) -> str: + return f"b2id://{self.file_id}" + + +def parse_uri(uri: str) -> Path | B2URI | B2FileIdURI: + parsed = urllib.parse.urlparse(uri) + if parsed.scheme == "": + return pathlib.Path(uri) + return _parse_b2_uri(uri, parsed) + + +def parse_b2_uri(uri: str) -> B2URI | B2FileIdURI: + parsed = urllib.parse.urlparse(uri) + return _parse_b2_uri(uri, parsed) + + +def _parse_b2_uri(uri, parsed: urllib.parse.ParseResult) -> B2URI | B2FileIdURI: + if parsed.scheme in ("b2", "b2id"): + if not parsed.netloc: + raise ValueError(f"Invalid B2 URI: {uri!r}") + elif parsed.password or parsed.username: + raise ValueError( + "Invalid B2 URI: credentials passed using `user@password:` syntax are not supported in URI" + ) + + if parsed.scheme == "b2": + return B2URI(bucket=parsed.netloc, path=parsed.path[1:]) + elif parsed.scheme == "b2id": + return B2FileIdURI(file_id=parsed.netloc) + else: + raise ValueError(f"Unsupported URI scheme: {parsed.scheme!r}") diff --git a/b2/arg_parser.py b/b2/arg_parser.py index 3f61ae3aa..45d60673f 100644 --- a/b2/arg_parser.py +++ b/b2/arg_parser.py @@ -9,6 +9,7 @@ ###################################################################### import argparse +import functools import locale import re import sys @@ -148,3 +149,20 @@ def parse_default_retention_period(s): 'default retention period must be in the form of "X days|years "' ) return RetentionPeriod(**{m.group('unit'): int(m.group('duration'))}) + + +def wrap_with_argument_type_error(func, translator=str, exc_type=ValueError): + """ + Wrap function that may raise an exception into a function that raises ArgumentTypeError error. + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + if isinstance(e, exc_type): + raise argparse.ArgumentTypeError(translator(e)) + raise + + return wrapper diff --git a/b2/console_tool.py b/b2/console_tool.py index c9f8d0eac..fa5103cc8 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -121,13 +121,15 @@ ) from b2._cli.obj_loads import validated_loads from b2._cli.shell import detect_shell -from b2._utils.filesystem import points_to_fifo +from b2._utils.filesystem import STDOUT_FILE_PATH, points_to_fifo +from b2._utils.uri import B2URI, B2FileIdURI, B2URIBase, parse_b2_uri from b2.arg_parser import ( ArgumentParser, parse_comma_separated_list, parse_default_retention_period, parse_millis_from_float_timestamp, parse_range, + wrap_with_argument_type_error, ) from b2.json_encoder import B2CliJsonEncoder from b2.version import VERSION @@ -202,6 +204,9 @@ def local_path_to_b2_path(path): return path.replace(os.path.sep, '/') +B2_URI_ARG_TYPE = wrap_with_argument_type_error(parse_b2_uri) + + def keyboard_interrupt_handler(signum, frame): raise KeyboardInterrupt() @@ -1412,18 +1417,27 @@ def _represent_legal_hold(cls, legal_hold: LegalHold): def _print_file_attribute(self, label, value): self._print((label + ':').ljust(20) + ' ' + value) + def get_local_output_filename(self, filename: str) -> str: + if filename == '-': + return STDOUT_FILE_PATH + return filename + @B2.register_subcommand class DownloadFileById( - ThreadsMixin, ProgressMixin, SourceSseMixin, WriteBufferSizeMixin, SkipHashVerificationMixin, - MaxDownloadStreamsMixin, DownloadCommand + ThreadsMixin, + ProgressMixin, + SourceSseMixin, + WriteBufferSizeMixin, + SkipHashVerificationMixin, + MaxDownloadStreamsMixin, + DownloadCommand, ): """ Downloads the given file, and stores it in the given local file. {PROGRESSMIXIN} {THREADSMIXIN} - {THREADSMIXIN} {SOURCESSEMIXIN} {WRITEBUFFERSIZEMIXIN} {SKIPHASHVERIFICATIONMIXIN} @@ -1442,9 +1456,8 @@ def _setup_parser(cls, parser): def run(self, args): super().run(args) - progress_listener = make_progress_listener( - args.localFileName, args.noProgress or args.quiet - ) + local_filename = self.get_local_output_filename(args.localFileName) + progress_listener = make_progress_listener(local_filename, args.noProgress or args.quiet) encryption_setting = self._get_source_sse_setting(args) self._set_threads_from_args(args) downloaded_file = self.api.download_file_by_id( @@ -1452,7 +1465,7 @@ def run(self, args): ) self._print_download_info(downloaded_file) - downloaded_file.save_to(args.localFileName) + downloaded_file.save_to(local_filename) self._print('Download finished') return 0 @@ -1460,8 +1473,8 @@ def run(self, args): @B2.register_subcommand class DownloadFileByName( - ProgressMixin, ThreadsMixin, + ProgressMixin, SourceSseMixin, WriteBufferSizeMixin, SkipHashVerificationMixin, @@ -1492,23 +1505,76 @@ def _setup_parser(cls, parser): def run(self, args): super().run(args) + local_filename = self.get_local_output_filename(args.localFileName) self._set_threads_from_args(args) bucket = self.api.get_bucket_by_name(args.bucketName) - progress_listener = make_progress_listener( - args.localFileName, args.noProgress or args.quiet - ) + progress_listener = make_progress_listener(local_filename, args.noProgress or args.quiet) encryption_setting = self._get_source_sse_setting(args) downloaded_file = bucket.download_file_by_name( args.b2FileName, progress_listener, encryption=encryption_setting ) self._print_download_info(downloaded_file) - downloaded_file.save_to(args.localFileName) + downloaded_file.save_to(local_filename) self._print('Download finished') return 0 +@B2.register_subcommand +class Cat( + ProgressMixin, + SourceSseMixin, + WriteBufferSizeMixin, + SkipHashVerificationMixin, + DownloadCommand, +): + """ + Download content of a file identified by B2 URI directly to stdout. + + {PROGRESSMIXIN} + {SOURCESSEMIXIN} + {WRITEBUFFERSIZEMIXIN} + {SKIPHASHVERIFICATIONMIXIN} + + Requires capability: + + - **readFiles** + """ + + @classmethod + def _setup_parser(cls, parser): + parser.add_argument( + 'b2uri', + type=B2_URI_ARG_TYPE, + help= + "B2 URI identifying the file to print, e.g. b2://yourBucket/file.txt or b2id://fileId", + ) + super()._setup_parser(parser) + + def download_by_b2_uri( + self, b2_uri: B2URIBase, args: argparse.Namespace, local_filename + ) -> DownloadedFile: + progress_listener = make_progress_listener(local_filename, args.noProgress or args.quiet) + encryption_setting = self._get_source_sse_setting(args) + if isinstance(b2_uri, B2FileIdURI): + download = functools.partial(self.api.download_file_by_id, b2_uri.file_id) + elif isinstance(b2_uri, B2URI): + bucket = self.api.get_bucket_by_name(b2_uri.bucket) + download = functools.partial(bucket.download_file_by_name, b2_uri.path) + else: # This should never happen since there are no more subclasses of B2URIBase + raise ValueError(f'Unsupported B2 URI: {b2_uri!r}') + + return download(progress_listener=progress_listener, encryption=encryption_setting) + + def run(self, args): + super().run(args) + local_filename = self.get_local_output_filename('-') + downloaded_file = self.download_by_b2_uri(args.b2uri, args, local_filename) + downloaded_file.save_to(local_filename) + return 0 + + @B2.register_subcommand class GetAccountInfo(Command): """ @@ -2913,7 +2979,7 @@ def get_input_stream(self, filename: str) -> 'str | int | io.BinaryIO': if filename == "-": if os.path.exists('-'): self._print_stderr( - "WARNING: Filename `-` won't be supported in the future and will be treated as stdin alias." + "WARNING: Filename `-` won't be supported in the future and will always be treated as stdin alias." ) else: return sys.stdin.buffer if platform.system() == "Windows" else sys.stdin.fileno() diff --git a/noxfile.py b/noxfile.py index 4b61124e6..5f28dee40 100644 --- a/noxfile.py +++ b/noxfile.py @@ -109,7 +109,7 @@ def install_myself(session, extras=None): session.run('pip', 'uninstall', 'b2sdk', '-y') session.run('python', 'setup.py', 'develop') os.chdir(cwd) - elif CI and not CD: + elif CI and not CD and False: # In CI, install B2 SDK from the master branch session.run( 'pip', 'install', 'git+https://github.com/Backblaze/b2-sdk-python.git#egg=b2sdk', diff --git a/requirements.txt b/requirements.txt index c9f4b6b6d..05a09bc7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ argcomplete>=2,<4 arrow>=1.0.2,<2.0.0 -b2sdk>=1.24.1,<2 +b2sdk @ git+https://github.com/reef-technologies/b2-sdk-python.git@8f652ee69df76d4a9972b1f42aec58b533c45783 docutils>=0.18.1 idna~=3.4; platform_system == 'Java' importlib-metadata~=3.3; python_version < '3.8' diff --git a/test/integration/conftest.py b/test/integration/conftest.py index ff336bdc8..a94ca504d 100755 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -170,12 +170,17 @@ def b2_tool(global_b2_tool): @pytest.fixture(autouse=True, scope='session') -def sample_file(): +def sample_filepath(): """Copy the README.md file to /tmp so that docker tests can access it""" - tmp_readme = pathlib.Path(f'{TEMPDIR}/README.md') + tmp_readme = pathlib.Path(TEMPDIR) / 'README.md' if not tmp_readme.exists(): tmp_readme.write_text((ROOT_PATH / 'README.md').read_text()) - return str(tmp_readme) + return tmp_readme + + +@pytest.fixture(autouse=True, scope='session') +def sample_file(sample_filepath): + return str(sample_filepath) @pytest.fixture(scope='session') diff --git a/test/integration/helpers.py b/test/integration/helpers.py index 3d6893855..bad6dfa78 100755 --- a/test/integration/helpers.py +++ b/test/integration/helpers.py @@ -20,8 +20,9 @@ import sys import threading import time +import warnings from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timedelta from os import environ, linesep, path from pathlib import Path from tempfile import gettempdir, mkdtemp, mktemp @@ -57,7 +58,7 @@ logger = logging.getLogger(__name__) -BUCKET_CLEANUP_PERIOD_MILLIS = 0 +BUCKET_CLEANUP_PERIOD_MILLIS = timedelta(hours=6).total_seconds() * 1000 ONE_HOUR_MILLIS = 60 * 60 * 1000 ONE_DAY_MILLIS = ONE_HOUR_MILLIS * 24 @@ -533,6 +534,11 @@ def list_file_versions(self, bucket_name): class TempDir: def __init__(self): + warnings.warn( + 'TempDir is deprecated; use pytest tmp_path fixture instead', + DeprecationWarning, + stacklevel=2, + ) self.dirpath = None def get_dir(self): @@ -549,15 +555,11 @@ def __exit__(self, exc_type, exc_val, exc_tb): def read_file(path: Union[str, Path]): - if isinstance(path, Path): - path = str(path) with open(path, 'rb') as f: return f.read() def write_file(path: Union[str, Path], contents: bytes): - if isinstance(path, Path): - path = str(path) with open(path, 'wb') as f: f.write(contents) diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index ea458203d..737a1ca30 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -59,20 +59,30 @@ def get_bucketinfo() -> Tuple[str, str]: return '--bucketInfo', json.dumps({BUCKET_CREATED_AT_MILLIS: str(current_time_millis())}), -def test_download(b2_tool, bucket_name, sample_file): +@pytest.fixture +def uploaded_sample_file(b2_tool, bucket_name, sample_filepath): + return b2_tool.should_succeed_json( + ['upload-file', '--quiet', bucket_name, + str(sample_filepath), 'sample_file'] + ) - uploaded_a = b2_tool.should_succeed_json( - ['upload-file', '--quiet', bucket_name, sample_file, 'a'] + +def test_download(b2_tool, bucket_name, sample_filepath, uploaded_sample_file, tmp_path): + output_a = tmp_path / 'a' + b2_tool.should_succeed( + [ + 'download-file-by-name', '--quiet', bucket_name, uploaded_sample_file['fileName'], + str(output_a) + ] ) - with TempDir() as dir_path: - b2_tool.should_succeed( - ['download-file-by-name', '--quiet', bucket_name, 'a', dir_path / 'a'] - ) - assert read_file(dir_path / 'a') == read_file(sample_file) - b2_tool.should_succeed( - ['download-file-by-id', '--quiet', uploaded_a['fileId'], dir_path / 'b'] - ) - assert read_file(dir_path / 'b') == read_file(sample_file) + assert output_a.read_text() == sample_filepath.read_text() + + output_b = tmp_path / 'b' + b2_tool.should_succeed( + ['download-file-by-id', '--quiet', uploaded_sample_file['fileId'], + str(output_b)] + ) + assert output_b.read_text() == sample_filepath.read_text() def test_basic(b2_tool, bucket_name, sample_file, is_running_on_docker): @@ -2677,3 +2687,22 @@ def test_upload_unbound_stream__redirect_operator( f'b2 upload-unbound-stream {bucket_name} <(echo -n {content}) {request.node.name}.txt' ) assert hashlib.sha1(content.encode()).hexdigest() in run.stdout + + +def test_download_file_stdout( + b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_file +): + assert b2_tool.should_succeed( + ['download-file-by-name', '--quiet', bucket_name, uploaded_sample_file['fileName'], '-'], + ) == sample_filepath.read_text() + assert b2_tool.should_succeed( + ['download-file-by-id', '--quiet', uploaded_sample_file['fileId'], '-'], + ) == sample_filepath.read_text() + + +def test_cat(b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_file): + assert b2_tool.should_succeed( + ['cat', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}"], + ) == sample_filepath.read_text() + assert b2_tool.should_succeed(['cat', f"b2id://{uploaded_sample_file['fileId']}"] + ) == sample_filepath.read_text() diff --git a/test/unit/_utils/test_uri.py b/test/unit/_utils/test_uri.py new file mode 100644 index 000000000..2605de653 --- /dev/null +++ b/test/unit/_utils/test_uri.py @@ -0,0 +1,72 @@ +###################################################################### +# +# File: test/unit/_utils/test_uri.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from pathlib import Path + +import pytest + +from b2._utils.uri import B2URI, B2FileIdURI, parse_uri + + +def test_b2pathuri_str(): + uri = B2URI(bucket="testbucket", path="/path/to/file") + assert str(uri) == "b2://testbucket/path/to/file" + + +def test_b2pathuri_is_dir_true(): + uri = B2URI(bucket="testbucket", path="/path/to/directory/") + assert uri.is_dir() is True + + +def test_b2pathuri_is_dir_false(): + uri = B2URI(bucket="testbucket", path="/path/to/file") + assert uri.is_dir() is False + + +def test_b2fileuri_str(): + uri = B2FileIdURI(file_id="file123") + assert str(uri) == "b2id://file123" + + +@pytest.mark.parametrize( + "uri,expected", + [ + ("some/local/path", Path("some/local/path")), + ("./some/local/path", Path("some/local/path")), + ("b2://bucket/path/to/dir/", B2URI(bucket="bucket", path="path/to/dir/")), + ("b2id://file123", B2FileIdURI(file_id="file123")), + ], +) +def test_parse_uri(uri, expected): + assert parse_uri(uri) == expected + + +@pytest.mark.parametrize( + "uri, expected_exception_message", + [ + # Test cases for invalid B2 URIs (missing netloc part) + ("b2://", "Invalid B2 URI: 'b2://'"), + ("b2id://", "Invalid B2 URI: 'b2id://'"), + # Test cases for B2 URIs with credentials + ( + "b2://user@password:bucket/path", + "Invalid B2 URI: credentials passed using `user@password:` syntax are not supported in URI", + ), + ( + "b2id://user@password:file123", + "Invalid B2 URI: credentials passed using `user@password:` syntax are not supported in URI", + ), + # Test cases for unsupported URI schemes + ("unknown://bucket/path", "Unsupported URI scheme: 'unknown'"), + ], +) +def test_parse_uri_exceptions(uri, expected_exception_message): + with pytest.raises(ValueError) as exc_info: + parse_uri(uri) + assert expected_exception_message in str(exc_info.value) diff --git a/test/unit/console_tool/test_download_file.py b/test/unit/console_tool/test_download_file.py index 63bdc60d7..5d80c0bdf 100644 --- a/test/unit/console_tool/test_download_file.py +++ b/test/unit/console_tool/test_download_file.py @@ -8,22 +8,25 @@ # ###################################################################### import os +import pathlib +from test.helpers import skip_on_windows +from test.unit.helpers import RunOrDieExecutor import pytest @pytest.fixture -def test_file_setup(tmpdir): +def local_file(tmp_path): """Set up a test file and return its path.""" filename = 'file1.txt' content = 'hello world' - local_file = tmpdir.join(filename) - local_file.write(content) + local_file = tmp_path / filename + local_file.write_text(content) mod_time = 1500111222 os.utime(local_file, (mod_time, mod_time)) - return local_file, content + return local_file EXPECTED_STDOUT_DOWNLOAD = ''' @@ -41,58 +44,126 @@ def test_file_setup(tmpdir): ''' -def upload_file(b2_cli, local_file, filename='file1.txt'): - """Helper function to upload a file.""" - b2_cli.run(['upload-file', 'my-bucket', str(local_file), filename]) - +@pytest.fixture +def uploaded_file(b2_cli, bucket, local_file): + filename = 'file1.txt' + b2_cli.run(['upload-file', bucket, str(local_file), filename]) + return { + 'bucket': bucket, + 'fileName': filename, + 'content': local_file.read_text(), + } -def test_download_file_by_name(b2_cli, bucket, test_file_setup): - local_file, content = test_file_setup - upload_file(b2_cli, local_file) +def test_download_file_by_name(b2_cli, local_file, uploaded_file, tmp_path): + output_path = tmp_path / 'output.txt' b2_cli.run( - ['download-file-by-name', '--noProgress', 'my-bucket', 'file1.txt', - str(local_file)], + [ + 'download-file-by-name', '--noProgress', uploaded_file['bucket'], + uploaded_file['fileName'], + str(output_path) + ], expected_stdout=EXPECTED_STDOUT_DOWNLOAD ) - assert local_file.read() == content + assert output_path.read_text() == uploaded_file['content'] -def test_download_file_by_name_quietly(b2_cli, bucket, test_file_setup): - local_file, content = test_file_setup - - upload_file(b2_cli, local_file) +def test_download_file_by_name_quietly(b2_cli, uploaded_file, tmp_path): + output_path = tmp_path / 'output.txt' b2_cli.run( - ['download-file-by-name', '--quiet', 'my-bucket', 'file1.txt', - str(local_file)], + [ + 'download-file-by-name', '--quiet', uploaded_file['bucket'], uploaded_file['fileName'], + str(output_path) + ], expected_stdout='' ) - assert local_file.read() == content - + assert output_path.read_text() == uploaded_file['content'] -def test_download_file_by_id(b2_cli, bucket, test_file_setup): - local_file, content = test_file_setup - upload_file(b2_cli, local_file) +def test_download_file_by_id(b2_cli, uploaded_file, tmp_path): + output_path = tmp_path / 'output.txt' b2_cli.run( ['download-file-by-id', '--noProgress', '9999', - str(local_file)], # <-- Here's the change + str(output_path)], expected_stdout=EXPECTED_STDOUT_DOWNLOAD ) - assert local_file.read() == content + assert output_path.read_text() == uploaded_file['content'] + + +def test_download_file_by_id_quietly(b2_cli, uploaded_file, tmp_path): + output_path = tmp_path / 'output.txt' + + b2_cli.run(['download-file-by-id', '--quiet', '9999', str(output_path)], expected_stdout='') + assert output_path.read_text() == uploaded_file['content'] + + +@skip_on_windows(reason='os.mkfifo is not supported on Windows') +def test_download_file_by_name__named_pipe(b2_cli, local_file, uploaded_file, tmp_path): + output_path = tmp_path / 'output.txt' + os.mkfifo(output_path) + + output_string = None + + def reader(): + nonlocal output_string + output_string = output_path.read_text() + + with RunOrDieExecutor() as executor: + reader_future = executor.submit(reader) + + b2_cli.run( + [ + 'download-file-by-name', '--noProgress', uploaded_file['bucket'], + uploaded_file['fileName'], + str(output_path) + ], + expected_stdout=EXPECTED_STDOUT_DOWNLOAD + ) + reader_future.result(timeout=1) + assert output_string == uploaded_file['content'] + + +@pytest.fixture +def uploaded_stdout_txt(b2_cli, bucket, local_file, tmp_path): + local_file.write_text('non-mocked /dev/stdout test ignore me') + b2_cli.run(['upload-file', bucket, str(local_file), 'stdout.txt']) + return { + 'bucket': bucket, + 'fileName': 'stdout.txt', + 'content': local_file.read_text(), + } + + +def test_download_file_by_name__to_stdout_by_alias( + b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd +): + """Test download_file_by_name stdout alias support""" + b2_cli.run( + ['download-file-by-name', '--noProgress', bucket, uploaded_stdout_txt['fileName'], '-'], + ) + assert capfd.readouterr().out == uploaded_stdout_txt['content'] + assert not pathlib.Path('-').exists() -def test_download_file_by_id_quietly(b2_cli, bucket, test_file_setup): - local_file, content = test_file_setup +def test_cat__b2_uri(b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd): + """Test download_file_by_name stdout alias support""" + b2_cli.run(['cat', '--noProgress', f"b2://{bucket}/{uploaded_stdout_txt['fileName']}"],) + assert capfd.readouterr().out == uploaded_stdout_txt['content'] - upload_file(b2_cli, local_file) +def test_cat__b2_uri__invalid(b2_cli, capfd): b2_cli.run( - ['download-file-by-id', '--quiet', '9999', - str(local_file)], # <-- Here's the change - expected_stdout='' + ['cat', "nothing/meaningful"], + expected_stderr=None, + expected_status=2, ) - assert local_file.read() == content + assert "argument b2uri: Unsupported URI scheme: ''" in capfd.readouterr().err + + +def test_cat__b2id_uri(b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd): + """Test download_file_by_name stdout alias support""" + b2_cli.run(['cat', '--noProgress', "b2id://9999"],) + assert capfd.readouterr().out == uploaded_stdout_txt['content'] diff --git a/test/unit/console_tool/test_upload_file.py b/test/unit/console_tool/test_upload_file.py index ff49a1af2..f485050ea 100644 --- a/test/unit/console_tool/test_upload_file.py +++ b/test/unit/console_tool/test_upload_file.py @@ -9,7 +9,7 @@ ###################################################################### import os from test.helpers import skip_on_windows -from test.unit.helpers import run_in_background +from test.unit.helpers import RunOrDieExecutor import b2 @@ -47,7 +47,7 @@ def test_upload_file__named_pipe(b2_cli, bucket, tmpdir): content = 'hello world' local_file1 = tmpdir.join('file1.txt') os.mkfifo(str(local_file1)) - writer = run_in_background( + writer = RunOrDieExecutor().submit( local_file1.write, content ) # writer will block until content is read @@ -66,13 +66,13 @@ def test_upload_file__named_pipe(b2_cli, bucket, tmpdir): remove_version=True, expected_part_of_stdout=expected_stdout, ) - writer.join() + writer.result(timeout=1) def test_upload_file__hyphen_file_instead_of_stdin(b2_cli, bucket, tmpdir, monkeypatch): """Test upload_file will upload file named `-` instead of stdin by default""" # TODO remove this in v4 - assert b2.__version__ < '4', "`-` file upload should not be supported in next major version of CLI" + assert b2.__version__ < '4', "`-` filename should not be supported in next major version of CLI" filename = 'stdin.txt' content = "I'm very rare creature, a file named '-'" monkeypatch.chdir(str(tmpdir)) @@ -92,7 +92,7 @@ def test_upload_file__hyphen_file_instead_of_stdin(b2_cli, bucket, tmpdir, monke remove_version=True, expected_part_of_stdout=expected_stdout, expected_stderr= - "WARNING: Filename `-` won't be supported in the future and will be treated as stdin alias.\n", + "WARNING: Filename `-` won't be supported in the future and will always be treated as stdin alias.\n", ) diff --git a/test/unit/console_tool/test_upload_unbound_stream.py b/test/unit/console_tool/test_upload_unbound_stream.py index 9c9015325..469105182 100644 --- a/test/unit/console_tool/test_upload_unbound_stream.py +++ b/test/unit/console_tool/test_upload_unbound_stream.py @@ -9,7 +9,7 @@ ###################################################################### import os from test.helpers import skip_on_windows -from test.unit.helpers import run_in_background +from test.unit.helpers import RunOrDieExecutor from b2._cli.const import DEFAULT_MIN_PART_SIZE @@ -21,7 +21,9 @@ def test_upload_unbound_stream__named_pipe(b2_cli, bucket, tmpdir): content = 'hello world' fifo_file = tmpdir.join('fifo_file.txt') os.mkfifo(str(fifo_file)) - writer = run_in_background(fifo_file.write, content) # writer will block until content is read + writer = RunOrDieExecutor().submit( + fifo_file.write, content + ) # writer will block until content is read expected_stdout = f'URL by file name: http://download.example.com/file/my-bucket/{filename}' expected_json = { @@ -37,7 +39,7 @@ def test_upload_unbound_stream__named_pipe(b2_cli, bucket, tmpdir): remove_version=True, expected_part_of_stdout=expected_stdout, ) - writer.join() + writer.result(timeout=1) def test_upload_unbound_stream__stdin(b2_cli, bucket, tmpdir, mock_stdin): @@ -72,8 +74,8 @@ def test_upload_unbound_stream__with_part_size_options(b2_cli, bucket, tmpdir, m filename = 'named_pipe.txt' fifo_file = tmpdir.join('fifo_file.txt') os.mkfifo(str(fifo_file)) - writer = run_in_background( - fifo_file.write, "x" * expected_size + writer = RunOrDieExecutor().submit( + lambda: fifo_file.write("x" * expected_size) ) # writer will block until content is read expected_stdout = f'URL by file name: http://download.example.com/file/my-bucket/{filename}' @@ -99,7 +101,7 @@ def test_upload_unbound_stream__with_part_size_options(b2_cli, bucket, tmpdir, m remove_version=True, expected_part_of_stdout=expected_stdout, ) - writer.join() + writer.result(timeout=1) def test_upload_unbound_stream__regular_file(b2_cli, bucket, tmpdir): diff --git a/test/unit/helpers.py b/test/unit/helpers.py index 62bfa3220..26e67659b 100644 --- a/test/unit/helpers.py +++ b/test/unit/helpers.py @@ -7,10 +7,30 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### -import threading +import concurrent.futures -def run_in_background(func, *args, **kwargs) -> threading.Thread: - thread = threading.Thread(target=func, args=args, kwargs=kwargs) - thread.start() - return thread +class RunOrDieExecutor(concurrent.futures.ThreadPoolExecutor): + """ + Deadly ThreadPoolExecutor, which ensures all task are quickly closed before exiting. + + Only really usable in tests. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._futures = [] + + def __exit__(self, exc_type, exc_val, exc_tb): + try: + self.shutdown(wait=False, cancel_futures=True) + except TypeError: # Python <3.9 + self.shutdown(wait=False) + for future in self._futures: + future.cancel() + return super().__exit__(exc_type, exc_val, exc_tb) + + def submit(self, *args, **kwargs): # to be removed when Python 3.9 is minimum + future = super().submit(*args, **kwargs) + self._futures.append(future) + return future diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 57aa6a606..285ba3312 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -134,6 +134,8 @@ def _remove_api_version_number(self, s): return re.sub(self.RE_API_VERSION, '/vx/', s) def _normalize_expected_output(self, text, format_vars=None): + if text is None: + return None format_vars = format_vars or {} return self._trim_leading_spaces(text).format( account_id=self.account_id, master_key=self.master_key, **format_vars @@ -213,7 +215,7 @@ def _run_command( ) print('EXPECTED TO FIND IN STDOUT:', repr(expected_part_of_stdout)) print('ACTUAL STDOUT: ', repr(actual_stdout)) - if expected_stderr != actual_stderr: + if expected_stderr is not None and expected_stderr != actual_stderr: print('EXPECTED STDERR:', repr(expected_stderr)) print('ACTUAL STDERR: ', repr(actual_stderr)) print(actual_stderr) @@ -235,8 +237,10 @@ def _run_command( self.assertIn(expected_part_of_stdout, actual_stdout) if unexpected_part_of_stdout is not None: self.assertNotIn(unexpected_part_of_stdout, actual_stdout) - self.assertEqual(expected_stderr, actual_stderr, 'stderr') + if expected_stderr is not None: + self.assertEqual(expected_stderr, actual_stderr, 'stderr') self.assertEqual(expected_status, actual_status, 'exit status code') + return actual_status, actual_stdout, actual_stderr @classmethod def _upload_multiple_files(cls, bucket):