refactor ls json streaming

b2/console_tool.py

@@ -685,27 +685,49 @@ def _parse_file_infos(cls, args_info):
             file_infos[parts[0]] = parts[1]
         return file_infos
 
-    def _print_json(self, data, *, raw: bool = False):
-        data = data if raw else json.dumps(data, indent=4, sort_keys=True, cls=B2CliJsonEncoder)
-        self._print(data, enforce_output=True, raw=raw)
+    def _print_json(self, data) -> None:
+        return self._print(
+            json.dumps(data, indent=4, sort_keys=True, cls=B2CliJsonEncoder), enforce_output=True
+        )
 
-    def _print(self, *args, enforce_output=False, raw: bool = False):
-        self._print_standard_descriptor(
-            self.stdout, 'stdout', *args, enforce_output=enforce_output, raw=raw
+    def _print(self, *args, enforce_output: bool = False, end: Optional[str] = None) -> None:
+        return self._print_standard_descriptor(
+            self.stdout, "stdout", *args, enforce_output=enforce_output, end=end
         )
 
-    def _print_stderr(self, *args, **kwargs):
-        self._print_standard_descriptor(self.stderr, 'stderr', *args, enforce_output=True)
+    def _print_stderr(self, *args, end: Optional[str] = None) -> None:
+        return self._print_standard_descriptor(
+            self.stderr, "stderr", *args, enforce_output=True, end=end
+        )
 
     def _print_standard_descriptor(
-        self, descriptor, descriptor_name, *args, enforce_output=False, raw: bool = False
-    ):
+        self,
+        descriptor,
+        descriptor_name: str,
+        *args,
+        enforce_output: bool = False,
+        end: Optional[str] = None,
+    ) -> None:
+        """
+        Prints to fd, unless quiet is set.
+
+        :param descriptor: file descriptor to print to
+        :param descriptor_name: name of the descriptor, used for error reporting
+        :param args: object to be printed
+        :param enforce_output: overrides quiet setting; Should not be used for anything other than data
+        :param end: end of the line characters; None for default newline
+        """
         if not self.quiet or enforce_output:
-            self._print_helper(descriptor, descriptor.encoding, descriptor_name, *args, raw=raw)
+            self._print_helper(descriptor, descriptor.encoding, descriptor_name, *args, end=end)
 
     @classmethod
     def _print_helper(
-        cls, descriptor, descriptor_encoding, descriptor_name, *args, raw: bool = False
+        cls,
+        descriptor,
+        descriptor_encoding: str,
+        descriptor_name: str,
+        *args,
+        end: Optional[str] = None
     ):
         try:
             descriptor.write(' '.join(args))
@@ -719,8 +741,7 @@ def _print_helper(
             args = [arg.encode('ascii', 'backslashreplace').decode() for arg in args]
             sys.stderr.write("Trying to print: %s\n" % args)
             descriptor.write(' '.join(args))
-        if not raw:
-            descriptor.write('\n')
+        descriptor.write("\n" if end is None else end)
 
     def __str__(self):
         return f'{self.__class__.__module__}.{self.__class__.__name__}'
@@ -1996,11 +2017,17 @@ def _setup_parser(cls, parser):
 
     def run(self, args):
         super().run(args)
-        self._print_files(args)
         if args.json:
-            if self._first_row:  # no files were printed
-                self._print_json('[', raw=True)
-            self._print_json(']\n', raw=True)
+            i = -1
+            for i, (file_version, _) in enumerate(self._get_ls_generator(args)):
+                if i:
+                    self._print(',', end='')
+                else:
+                    self._print('[')
+                self._print_json(file_version)
+            self._print(']' if i >= 0 else '[]')
+        else:
+            self._print_files(args)
         return 0
 
     def _print_file_version(
@@ -2222,6 +2249,7 @@ def _setup_parser(cls, parser):
         super()._setup_parser(parser)
 
     def run(self, args):
+        super().run(args)
         if args.dryRun:
             self._print_files(args)
             return 0

test/integration/helpers.py

@@ -337,7 +337,7 @@ def should_equal(expected, actual):
 class CommandLine:
 
     EXPECTED_STDERR_PATTERNS = [
-        re.compile(r'^Using https?://[\w.]+.com$'),  # account auth
+        re.compile(r'^Using https?://[\w.]+$'),  # account auth
         re.compile(r'.*B/s]$', re.DOTALL),  # progress bar
         re.compile(r'^\r?$'),  # empty line
         re.compile(

test/unit/test_console_tool.py

@@ -84,9 +84,7 @@ def _run_command_ignore_output(self, argv):
             print('ACTUAL STDERR:  ', repr(actual_stderr))
             print(actual_stderr)
 
-        assert re.match(
-            r'^(|Using https?://[\w.]+.com\n)$', actual_stderr
-        ), f"stderr: {actual_stderr!r}"
+        assert re.match(r'^(|Using https?://[\w.]+)$', actual_stderr), f"stderr: {actual_stderr!r}"
         self.assertEqual(0, actual_status, 'exit status code')
 
     def _trim_leading_spaces(self, s):
@@ -287,9 +285,11 @@ def test_authorize_with_good_key_using_underscore(self):
         assert self.account_info.get_account_auth_token() is None
 
         # Authorize an account with a good api key.
+        expected_stderr = """
+        Using http://production.example.com
+        """
         self._run_command(
-            ['authorize_account', self.account_id, self.master_key], '',
-            "Using http://production.example.com\n", 0
+            ['authorize_account', self.account_id, self.master_key], '', expected_stderr, 0
         )
 
         # Auth token should be in account info now
@@ -339,15 +339,17 @@ def test_authorize_towards_custom_realm(self):
         # Auth token should be in account info now
         assert self.account_info.get_account_auth_token() is not None
 
+        expected_stderr = """
+        Using http://custom2.example.com
+        """
         # realm provided with env var
         with mock.patch.dict(
             'os.environ', {
                 B2_ENVIRONMENT_ENV_VAR: 'http://custom2.example.com',
             }
         ):
             self._run_command(
-                ['authorize-account', self.account_id, self.master_key], '',
-                'Using http://custom2.example.com\n', 0
+                ['authorize-account', self.account_id, self.master_key], '', expected_stderr, 0
             )
 
     def test_create_key_and_authorize_with_it(self):