Download to stdout & cat command

CHANGELOG.md

@@ -6,11 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+* Add `cat` command for downloading file contents directly to stdout
+
 ### Fixed
 * Emit `Using https://api.backblazeb2.com` message to stderr instead of stdout, therefor prevent JSON output corruption
 
 ### Changed
 * Stream `ls --json` JSON output instead of dumping it only after all objects have been fetched
+* Alias `-` to stdout in `download-file-by-name` or `download-file-by-id` command
 
 ## [3.12.0] - 2023-10-28
 
@@ -58,7 +62,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Add s3 endpoint to `get-account-info`
 
 ### Deprecated
-* Support of `-` as a valid filename in `upload-file` command. In future `-` will be an alias for standard input.
+* Deprecate support of `-` as a valid filename in `upload-file` command. In the future `-` will always be interpreted as standard input
 
 ### Changed
 * Better help text for --corsRules

b2/_utils/filesystem.py

@@ -7,6 +7,7 @@
 # License https://www.backblaze.com/using_b2_code.html
 #
 ######################################################################
+import platform
 import stat
 from pathlib import Path
 
@@ -18,3 +19,6 @@ def points_to_fifo(path: Path) -> bool:
         return stat.S_ISFIFO(path.stat().st_mode)
     except OSError:
         return False
+
+
+STDOUT_FILE_PATH = "CON" if platform.system() == "Windows" else "/dev/stdout"

b2/_utils/uri.py

@@ -0,0 +1,76 @@
+######################################################################
+#
+# File: b2/_utils/uri.py
+#
+# Copyright 2023 Backblaze Inc. All Rights Reserved.
+#
+# License https://www.backblaze.com/using_b2_code.html
+#
+######################################################################
+from __future__ import annotations
+
+import dataclasses
+import pathlib
+import urllib
+from pathlib import Path
+
+
+class B2URIBase:
+    pass
+
+
+@dataclasses.dataclass
+class B2URI(B2URIBase):
+    bucket: str
+    path: str
+
+    def __str__(self) -> str:
+        return f"b2://{self.bucket}{self.path}"
+
+    def is_dir(self) -> bool:
+        """
+        Return if the path is a directory.
+
+        Please note this is symbolical.
+        It is possible for file to have a trailing slash, but it is HIGHLY discouraged, and not supported by B2 CLI.
+
+        :return: True if the path is a file, False if it's a directory
+        """
+        return self.path.endswith("/")
+
+
+@dataclasses.dataclass
+class B2FileIdURI(B2URIBase):
+    file_id: str
+
+    def __str__(self) -> str:
+        return f"b2id://{self.file_id}"
+
+
+def parse_uri(uri: str) -> Path | B2URI | B2FileIdURI:
+    parsed = urllib.parse.urlparse(uri)
+    if parsed.scheme == "":
+        return pathlib.Path(uri)
+    return _parse_b2_uri(uri, parsed)
+
+
+def parse_b2_uri(uri: str) -> B2URI | B2FileIdURI:
+    parsed = urllib.parse.urlparse(uri)
+    return _parse_b2_uri(uri, parsed)
+
+
+def _parse_b2_uri(uri, parsed: urllib.parse.ParseResult) -> B2URI | B2FileIdURI:
+    if parsed.scheme in ("b2", "b2id"):
+        if not parsed.netloc:
+            raise ValueError(f"Invalid B2 URI: {uri!r}")
+        elif parsed.password or parsed.username:
+            raise ValueError(
+                "Invalid B2 URI: credentials passed using `user@password:` syntax are not supported in URI"
+            )
+
+        if parsed.scheme == "b2":
+            return B2URI(bucket=parsed.netloc, path=parsed.path[1:])
+        elif parsed.scheme == "b2id":
+            return B2FileIdURI(file_id=parsed.netloc)
+    else:
+        raise ValueError(f"Unsupported URI scheme: {parsed.scheme!r}")

b2/arg_parser.py

@@ -9,6 +9,7 @@
 ######################################################################
 
 import argparse
+import functools
 import locale
 import re
 import sys
@@ -148,3 +149,20 @@ def parse_default_retention_period(s):
             'default retention period must be in the form of "X days|years "'
         )
     return RetentionPeriod(**{m.group('unit'): int(m.group('duration'))})
+
+
+def wrap_with_argument_type_error(func, translator=str, exc_type=ValueError):
+    """
+    Wrap function that may raise an exception into a function that raises ArgumentTypeError error.
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            if isinstance(e, exc_type):
+                raise argparse.ArgumentTypeError(translator(e))
+            raise
+
+    return wrapper

b2/console_tool.py

@@ -121,13 +121,15 @@
 )
 from b2._cli.obj_loads import validated_loads
 from b2._cli.shell import detect_shell
-from b2._utils.filesystem import points_to_fifo
+from b2._utils.filesystem import STDOUT_FILE_PATH, points_to_fifo
+from b2._utils.uri import B2URI, B2FileIdURI, B2URIBase, parse_b2_uri
 from b2.arg_parser import (
     ArgumentParser,
     parse_comma_separated_list,
     parse_default_retention_period,
     parse_millis_from_float_timestamp,
     parse_range,
+    wrap_with_argument_type_error,
 )
 from b2.json_encoder import B2CliJsonEncoder
 from b2.version import VERSION
@@ -202,6 +204,9 @@ def local_path_to_b2_path(path):
     return path.replace(os.path.sep, '/')
 
 
+B2_URI_ARG_TYPE = wrap_with_argument_type_error(parse_b2_uri)
+
+
 def keyboard_interrupt_handler(signum, frame):
     raise KeyboardInterrupt()
 
@@ -1412,18 +1417,27 @@ def _represent_legal_hold(cls, legal_hold: LegalHold):
     def _print_file_attribute(self, label, value):
         self._print((label + ':').ljust(20) + ' ' + value)
 
+    def get_local_output_filename(self, filename: str) -> str:
+        if filename == '-':
+            return STDOUT_FILE_PATH
+        return filename
+
 
 @B2.register_subcommand
 class DownloadFileById(
-    ThreadsMixin, ProgressMixin, SourceSseMixin, WriteBufferSizeMixin, SkipHashVerificationMixin,
-    MaxDownloadStreamsMixin, DownloadCommand
+    ThreadsMixin,
+    ProgressMixin,
+    SourceSseMixin,
+    WriteBufferSizeMixin,
+    SkipHashVerificationMixin,
+    MaxDownloadStreamsMixin,
+    DownloadCommand,
 ):
     """
     Downloads the given file, and stores it in the given local file.
 
     {PROGRESSMIXIN}
     {THREADSMIXIN}
-    {THREADSMIXIN}
     {SOURCESSEMIXIN}
     {WRITEBUFFERSIZEMIXIN}
     {SKIPHASHVERIFICATIONMIXIN}
@@ -1442,26 +1456,25 @@ def _setup_parser(cls, parser):
 
     def run(self, args):
         super().run(args)
-        progress_listener = make_progress_listener(
-            args.localFileName, args.noProgress or args.quiet
-        )
+        local_filename = self.get_local_output_filename(args.localFileName)
+        progress_listener = make_progress_listener(local_filename, args.noProgress or args.quiet)
         encryption_setting = self._get_source_sse_setting(args)
         self._set_threads_from_args(args)
         downloaded_file = self.api.download_file_by_id(
             args.fileId, progress_listener, encryption=encryption_setting
         )
 
         self._print_download_info(downloaded_file)
-        downloaded_file.save_to(args.localFileName)
+        downloaded_file.save_to(local_filename)
         self._print('Download finished')
 
         return 0
 
 
 @B2.register_subcommand
 class DownloadFileByName(
-    ProgressMixin,
     ThreadsMixin,
+    ProgressMixin,
     SourceSseMixin,
     WriteBufferSizeMixin,
     SkipHashVerificationMixin,
@@ -1492,23 +1505,76 @@ def _setup_parser(cls, parser):
 
     def run(self, args):
         super().run(args)
+        local_filename = self.get_local_output_filename(args.localFileName)
         self._set_threads_from_args(args)
         bucket = self.api.get_bucket_by_name(args.bucketName)
-        progress_listener = make_progress_listener(
-            args.localFileName, args.noProgress or args.quiet
-        )
+        progress_listener = make_progress_listener(local_filename, args.noProgress or args.quiet)
         encryption_setting = self._get_source_sse_setting(args)
         downloaded_file = bucket.download_file_by_name(
             args.b2FileName, progress_listener, encryption=encryption_setting
         )
 
         self._print_download_info(downloaded_file)
-        downloaded_file.save_to(args.localFileName)
+        downloaded_file.save_to(local_filename)
         self._print('Download finished')
 
         return 0
 
 
+@B2.register_subcommand
+class Cat(
+    ProgressMixin,
+    SourceSseMixin,
+    WriteBufferSizeMixin,
+    SkipHashVerificationMixin,
+    DownloadCommand,
+):
+    """
+    Download content of a file identified by B2 URI directly to stdout.
+
+    {PROGRESSMIXIN}
+    {SOURCESSEMIXIN}
+    {WRITEBUFFERSIZEMIXIN}
+    {SKIPHASHVERIFICATIONMIXIN}
+
+    Requires capability:
+
+    - **readFiles**
+    """
+
+    @classmethod
+    def _setup_parser(cls, parser):
+        parser.add_argument(
+            'b2uri',
+            type=B2_URI_ARG_TYPE,
+            help=
+            "B2 URI identifying the file to print, e.g. b2://yourBucket/file.txt or b2id://fileId",
+        )
+        super()._setup_parser(parser)
+
+    def download_by_b2_uri(
+        self, b2_uri: B2URIBase, args: argparse.Namespace, local_filename
+    ) -> DownloadedFile:
+        progress_listener = make_progress_listener(local_filename, args.noProgress or args.quiet)
+        encryption_setting = self._get_source_sse_setting(args)
+        if isinstance(b2_uri, B2FileIdURI):
+            download = functools.partial(self.api.download_file_by_id, b2_uri.file_id)
+        elif isinstance(b2_uri, B2URI):
+            bucket = self.api.get_bucket_by_name(b2_uri.bucket)
+            download = functools.partial(bucket.download_file_by_name, b2_uri.path)
+        else:  # This should never happen since there are no more subclasses of B2URIBase
+            raise ValueError(f'Unsupported B2 URI: {b2_uri!r}')
+
+        return download(progress_listener=progress_listener, encryption=encryption_setting)
+
+    def run(self, args):
+        super().run(args)
+        local_filename = self.get_local_output_filename('-')
+        downloaded_file = self.download_by_b2_uri(args.b2uri, args, local_filename)
+        downloaded_file.save_to(local_filename)
+        return 0
+
+
 @B2.register_subcommand
 class GetAccountInfo(Command):
     """
@@ -2913,7 +2979,7 @@ def get_input_stream(self, filename: str) -> 'str | int | io.BinaryIO':
         if filename == "-":
             if os.path.exists('-'):
                 self._print_stderr(
-                    "WARNING: Filename `-` won't be supported in the future and will be treated as stdin alias."
+                    "WARNING: Filename `-` won't be supported in the future and will always be treated as stdin alias."
                 )
             else:
                 return sys.stdin.buffer if platform.system() == "Windows" else sys.stdin.fileno()

noxfile.py

@@ -109,7 +109,7 @@ def install_myself(session, extras=None):
         session.run('pip', 'uninstall', 'b2sdk', '-y')
         session.run('python', 'setup.py', 'develop')
         os.chdir(cwd)
-    elif CI and not CD:
+    elif CI and not CD and False:
         # In CI, install B2 SDK from the master branch
         session.run(
             'pip', 'install', 'git+https://github.com/Backblaze/b2-sdk-python.git#egg=b2sdk',

requirements.txt

@@ -1,6 +1,6 @@
 argcomplete>=2,<4
 arrow>=1.0.2,<2.0.0
-b2sdk>=1.24.1,<2
+b2sdk @ git+https://github.com/reef-technologies/b2-sdk-python.git@8f652ee69df76d4a9972b1f42aec58b533c45783
 docutils>=0.18.1
 idna~=3.4; platform_system == 'Java'
 importlib-metadata~=3.3; python_version < '3.8'

test/integration/conftest.py

@@ -170,12 +170,17 @@ def b2_tool(global_b2_tool):
 
 
 @pytest.fixture(autouse=True, scope='session')
-def sample_file():
+def sample_filepath():
     """Copy the README.md file to /tmp so that docker tests can access it"""
-    tmp_readme = pathlib.Path(f'{TEMPDIR}/README.md')
+    tmp_readme = pathlib.Path(TEMPDIR) / 'README.md'
     if not tmp_readme.exists():
         tmp_readme.write_text((ROOT_PATH / 'README.md').read_text())
-    return str(tmp_readme)
+    return tmp_readme
+
+
+@pytest.fixture(autouse=True, scope='session')
+def sample_file(sample_filepath):
+    return str(sample_filepath)
 
 
 @pytest.fixture(scope='session')