Merge pull request #1009 from reef-technologies/inmemory-if-env-vars-…

…auth

Use `InMemoryAccountInfo` when using env vars for authentication

b2/_internal/_cli/b2api.py

@@ -15,13 +15,26 @@
     AuthInfoCache,
     B2Api,
     B2HttpApiConfig,
+    InMemoryAccountInfo,
+    InMemoryCache,
     SqliteAccountInfo,
 )
+from b2sdk.v2.exception import MissingAccountData
 
 from b2._internal._cli.const import B2_USER_AGENT_APPEND_ENV_VAR
 
 
-def _get_b2api_for_profile(profile: Optional[str] = None, **kwargs) -> B2Api:
+def _get_b2api_for_profile(
+    profile: Optional[str] = None,
+    raise_if_does_not_exist: bool = False,
+    **kwargs,
+) -> B2Api:
+
+    if raise_if_does_not_exist:
+        account_info_file = SqliteAccountInfo._get_user_account_info_path(profile=profile)
+        if not os.path.exists(account_info_file):
+            raise MissingAccountData(account_info_file)
+
     account_info = SqliteAccountInfo(profile=profile)
     b2api = B2Api(
         api_config=_get_b2httpapiconfig(),
@@ -46,5 +59,9 @@ def _get_b2api_for_profile(profile: Optional[str] = None, **kwargs) -> B2Api:
     return b2api
 
 
+def _get_inmemory_b2api(**kwargs) -> B2Api:
+    return B2Api(InMemoryAccountInfo(), cache=InMemoryCache(), **kwargs)
+
+
 def _get_b2httpapiconfig():
     return B2HttpApiConfig(user_agent_append=os.environ.get(B2_USER_AGENT_APPEND_ENV_VAR),)

b2/_internal/_cli/b2args.py

@@ -12,9 +12,15 @@
 """
 import argparse
 import functools
+from os import environ
+from typing import Optional, Tuple
 
 from b2._internal._cli.arg_parser_types import wrap_with_argument_type_error
 from b2._internal._cli.argcompleters import b2uri_file_completer
+from b2._internal._cli.const import (
+    B2_APPLICATION_KEY_ENV_VAR,
+    B2_APPLICATION_KEY_ID_ENV_VAR,
+)
 from b2._internal._utils.uri import B2URI, B2URIBase, parse_b2_uri
 
 
@@ -76,3 +82,7 @@ def add_b2id_or_file_like_b2_uri_argument(parser: argparse.ArgumentParser, name=
         type=B2ID_OR_FILE_LIKE_B2_URI_ARG_TYPE,
         help="B2 URI pointing to a file, e.g. b2://yourBucket/file.txt or b2id://fileId",
     ).completer = b2uri_file_completer
+
+
+def get_keyid_and_key_from_env_vars() -> Tuple[Optional[str], Optional[str]]:
+    return environ.get(B2_APPLICATION_KEY_ID_ENV_VAR), environ.get(B2_APPLICATION_KEY_ENV_VAR)

b2/_internal/b2v3/registry.py

@@ -10,12 +10,40 @@
 
 # ruff: noqa: F405
 from b2._internal._b2v4.registry import *  # noqa
+from b2._internal._cli.b2api import _get_b2api_for_profile
 from b2._internal.arg_parser import enable_camel_case_arguments
 from .rm import Rm
 
 enable_camel_case_arguments()
 
 
+class ConsoleTool(ConsoleTool):
+    # same as original console tool, but does not use InMemoryAccountInfo and InMemoryCache
+    # when auth env vars are used
+
+    @classmethod
+    def _initialize_b2_api(cls, args: argparse.Namespace, kwargs: dict) -> B2Api:
+        return _get_b2api_for_profile(profile=args.profile, **kwargs)
+
+
+def main() -> None:
+    # this is a copy of v4 `main()` but with custom console tool class
+
+    ct = ConsoleTool(stdout=sys.stdout, stderr=sys.stderr)
+    exit_status = ct.run_command(sys.argv)
+    logger.info('\\\\ %s %s %s //', SEPARATOR, ('exit=%s' % exit_status).center(8), SEPARATOR)
+
+    # I haven't tracked down the root cause yet, but in Python 2.7, the futures
+    # packages is hanging on exit sometimes, waiting for a thread to finish.
+    # This happens when using sync to upload files.
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    logging.shutdown()
+
+    os._exit(exit_status)
+
+
 class Ls(B2URIBucketNFolderNameArgMixin, BaseLs):
     """
     {BaseLs}

b2/_internal/console_tool.py

@@ -130,10 +130,11 @@
     AutocompleteInstallError,
     autocomplete_install,
 )
-from b2._internal._cli.b2api import _get_b2api_for_profile
+from b2._internal._cli.b2api import _get_b2api_for_profile, _get_inmemory_b2api
 from b2._internal._cli.b2args import (
     add_b2id_or_b2_uri_argument,
     add_b2id_or_file_like_b2_uri_argument,
+    get_keyid_and_key_from_env_vars,
 )
 from b2._internal._cli.const import (
     B2_APPLICATION_KEY_ENV_VAR,
@@ -4179,11 +4180,11 @@ class ConsoleTool:
     Implements the commands available in the B2 command-line tool
     using the B2Api library.
 
-    Uses a ``b2sdk.SqlitedAccountInfo`` object to keep account data between runs.
+    Uses a ``b2sdk.SqlitedAccountInfo`` object to keep account data between runs
+    (unless authorization is performed via environment variables).
     """
 
-    def __init__(self, b2_api: B2Api | None, stdout, stderr):
-        self.api = b2_api
+    def __init__(self, stdout, stderr):
         self.stdout = stdout
         self.stderr = stderr
         self.b2_binary_name = 'b2'
@@ -4214,32 +4215,16 @@ def run_command(self, argv):
             # in case any control characters slip through escaping, just delete them
             self.stdout = NoControlCharactersStdout(self.stdout)
             self.stderr = NoControlCharactersStdout(self.stderr)
-        if self.api:
-            if (
-                args.profile or getattr(args, 'write_buffer_size', None) or
-                getattr(args, 'skip_hash_verification', None) or
-                getattr(args, 'max_download_streams_per_file', None)
-            ):
-                self._print_stderr(
-                    'ERROR: cannot change configuration on already initialized object'
-                )
-                return 1
-
-        else:
-            kwargs = {
-                'profile': args.profile,
-            }
-
-            if 'write_buffer_size' in args:
-                kwargs['save_to_buffer_size'] = args.write_buffer_size
-
-            if 'skip_hash_verification' in args:
-                kwargs['check_download_hash'] = not args.skip_hash_verification
 
-            if 'max_download_streams_per_file' in args:
-                kwargs['max_download_streams_per_file'] = args.max_download_streams_per_file
+        kwargs = {}
+        with suppress(AttributeError):
+            kwargs['save_to_buffer_size'] = args.write_buffer_size
+        with suppress(AttributeError):
+            kwargs['check_download_hash'] = not args.skip_hash_verification
+        with suppress(AttributeError):
+            kwargs['max_download_streams_per_file'] = args.max_download_streams_per_file
 
-            self.api = _get_b2api_for_profile(**kwargs)
+        self.api = self._initialize_b2_api(args=args, kwargs=kwargs)
 
         b2_command = B2(self)
         command_class = b2_command.run(args)
@@ -4251,9 +4236,10 @@ def run_command(self, argv):
             logger.info('starting command [%s] with arguments: %s', command, argv)
 
         try:
-            auth_ret = self.authorize_from_env(command_class)
-            if auth_ret:
-                return auth_ret
+            if command_class.REQUIRES_AUTH:
+                auth_ret = self.authorize_from_env()
+                if auth_ret:
+                    return auth_ret
             return command.run(args)
         except MissingAccountData as e:
             logger.exception('ConsoleTool missing account data error')
@@ -4274,12 +4260,34 @@ def run_command(self, argv):
             logger.exception('ConsoleTool unexpected exception')
             raise
 
-    def authorize_from_env(self, command_class):
-        if not command_class.REQUIRES_AUTH:
-            return 0
+    @classmethod
+    def _initialize_b2_api(cls, args: argparse.Namespace, kwargs: dict) -> B2Api:
+        b2_api = None
+        key_id, key = get_keyid_and_key_from_env_vars()
+        if key_id and key:
+            try:
+                # here we initialize regular b2 api on disk and check whether it matches
+                # the keys from env vars; if they indeed match then there's no need to
+                # initialize in-memory account info cause it's already stored on disk
+                b2_api = _get_b2api_for_profile(
+                    profile=args.profile, raise_if_does_not_exist=True, **kwargs
+                )
+                realm = os.environ.get(B2_ENVIRONMENT_ENV_VAR) or 'production'
+                is_same_key_on_disk = b2_api.account_info.is_same_key(key_id, realm)
+            except MissingAccountData:
+                is_same_key_on_disk = False
+
+            if not is_same_key_on_disk and args.command_class not in (
+                AuthorizeAccount, ClearAccount
+            ):
+                # when user specifies keys via env variables, we switch to in-memory account info
+                return _get_inmemory_b2api(**kwargs)
+
+        return b2_api or _get_b2api_for_profile(profile=args.profile, **kwargs)
+
+    def authorize_from_env(self) -> int:
 
-        key_id = os.environ.get(B2_APPLICATION_KEY_ID_ENV_VAR)
-        key = os.environ.get(B2_APPLICATION_KEY_ENV_VAR)
+        key_id, key = get_keyid_and_key_from_env_vars()
 
         if key_id is None and key is None:
             return 0
@@ -4349,7 +4357,7 @@ def _setup_logging(cls, args, argv):
 
 
 def main():
-    ct = ConsoleTool(b2_api=None, stdout=sys.stdout, stderr=sys.stderr)
+    ct = ConsoleTool(stdout=sys.stdout, stderr=sys.stderr)
     exit_status = ct.run_command(sys.argv)
     logger.info('\\\\ %s %s %s //', SEPARATOR, ('exit=%s' % exit_status).center(8), SEPARATOR)
 

changelog.d/+not-saving-auth-data-when-env-vars.fixed.md

@@ -0,0 +1 @@
+Don't persist credentials provided in the Enviornment variables in any command other than `authorize-account` when using `b2v4`.

test/integration/conftest.py

@@ -100,13 +100,18 @@ def get_raw_cli_int_version(config) -> int | None:
     return None
 
 
+def get_cli_int_version(config) -> int:
+    return get_raw_cli_int_version(config) or get_int_version(LATEST_STABLE_VERSION)
+
+
 @pytest.fixture(scope='session')
-def apiver(request):
-    return f"v{get_cli_int_version(request.config)}"
+def apiver_int(request):
+    return get_cli_int_version(request.config)
 
 
-def get_cli_int_version(config) -> int:
-    return get_raw_cli_int_version(config) or get_int_version(LATEST_STABLE_VERSION)
+@pytest.fixture(scope='session')
+def apiver(apiver_int):
+    return f"v{apiver_int}"
 
 
 @pytest.hookimpl
@@ -205,11 +210,12 @@ def monkeysession():
 
 
 @pytest.fixture(scope='session', autouse=True)
-def auto_change_account_info_dir(monkeysession) -> dir:
+def auto_change_account_info_dir(monkeysession) -> str:
     """
     Automatically for the whole testing:
     1) temporary remove B2_APPLICATION_KEY and B2_APPLICATION_KEY_ID from environment
     2) create a temporary directory for storing account info database
+    3) set B2_ACCOUNT_INFO_ENV_VAR to point to the temporary account info file
     """
 
     monkeysession.delenv('B2_APPLICATION_KEY_ID', raising=False)
@@ -269,6 +275,11 @@ def b2_tool(global_b2_tool):
     return global_b2_tool
 
 
+@pytest.fixture
+def account_info_file() -> pathlib.Path:
+    return pathlib.Path(os.environ[B2_ACCOUNT_INFO_ENV_VAR]).expanduser()
+
+
 @pytest.fixture
 def schedule_bucket_cleanup(global_b2_tool):
     """

test/integration/helpers.py

@@ -27,9 +27,9 @@
 import warnings
 from dataclasses import dataclass
 from datetime import datetime, timedelta
-from os import environ, linesep, path
+from os import environ, linesep
 from pathlib import Path
-from tempfile import gettempdir, mkdtemp, mktemp
+from tempfile import mkdtemp, mktemp
 
 import backoff
 from b2sdk.v2 import (
@@ -47,7 +47,6 @@
     InMemoryCache,
     LegalHold,
     RetentionMode,
-    SqliteAccountInfo,
     fix_windows_path_limit,
 )
 from b2sdk.v2.exception import (
@@ -336,33 +335,6 @@ def read_from(self, f):
             self.string = str(e)
 
 
-class EnvVarTestContext:
-    """
-    Establish config for environment variable test.
-    Copy the B2 credential file and rename the existing copy
-    """
-    ENV_VAR = 'B2_ACCOUNT_INFO'
-
-    def __init__(self, account_info_file_name: str):
-        self.account_info_file_name = account_info_file_name
-        self.suffix = ''.join(RNG.choice('abcdefghijklmnopqrstuvwxyz0123456789') for _ in range(7))
-
-    def __enter__(self):
-        src = self.account_info_file_name
-        dst = path.join(gettempdir(), 'b2_account_info')
-        shutil.copyfile(src, dst)
-        shutil.move(src, src + self.suffix)
-        environ[self.ENV_VAR] = dst
-        return dst
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        os.remove(environ.get(self.ENV_VAR))
-        fname = self.account_info_file_name
-        shutil.move(fname + self.suffix, fname)
-        if environ.get(self.ENV_VAR) is not None:
-            del environ[self.ENV_VAR]
-
-
 def should_equal(expected, actual):
     print('  expected:')
     print_json_indented(expected)
@@ -406,7 +378,6 @@ def __init__(
         self.realm = realm
         self.bucket_name_prefix = bucket_name_prefix
         self.env_file_cmd_placeholder = env_file_cmd_placeholder
-        self.env_var_test_context = EnvVarTestContext(SqliteAccountInfo().filename)
         self.api_wrapper = api_wrapper
         self.b2_uri_args = b2_uri_args