Convert CLI tests to use internal interface

This is significantly faster and provides more accurate test coverage
than running the commands as subprocessses.

src/CSET/__init__.py

@@ -25,14 +25,14 @@
 from CSET._common import ArgumentError
 
 
-def main():
+def main(raw_cli_args: list[str] = sys.argv):
     """CLI entrypoint.
 
     Handles argument parsing, setting up logging, top level error capturing,
     and execution of the desired subcommand.
     """
     parser = setup_argument_parser()
-    cli_args = sys.argv[1:] + shlex.split(os.getenv("CSET_ADDOPTS", ""))
+    cli_args = raw_cli_args[1:] + shlex.split(os.getenv("CSET_ADDOPTS", ""))
     args, unparsed_args = parser.parse_known_args(cli_args)
     setup_logging(args.verbose)
 

tests/test_cli.py

@@ -17,63 +17,153 @@
 In many ways these are integration tests.
 """
 
-import os
+import logging
 import subprocess
 from pathlib import Path
 from uuid import uuid4
 
 import pytest
 
+import CSET
+import CSET.operators
 
-def test_command_line_help():
-    """Check that help commands work."""
-    subprocess.run(["cset", "--help"], check=True)
+
+def test_command_line_invocation():
+    """Check invocation via different entrypoints.
+
+    Uses subprocess to validate the external interface.
+    """
+    # Invoke via cset command
     subprocess.run(["cset", "--version"], check=True)
+    # Invoke via __main__.py
+    subprocess.run(["python3", "-m", "CSET", "--version"], check=True)
+
+
+# Every other test should not use the command line interface, but rather stay
+# within python to ensure coverage measurement.
+def test_argument_parser():
+    """Tests the argument parser behaves appropriately."""
+    parser = CSET.setup_argument_parser()
+    # Test verbose flag.
+    args = parser.parse_args(["recipe-id", "-r", "recipe.yaml"])
+    assert args.verbose == 0
+    args = parser.parse_args(["-v", "recipe-id", "-r", "recipe.yaml"])
+    assert args.verbose == 1
+    args = parser.parse_args(["-vv", "recipe-id", "-r", "recipe.yaml"])
+    assert args.verbose == 2
+
+
+def test_setup_logging():
+    """Tests the logging setup at various verbosity levels."""
+    root_logger = logging.getLogger()
+    # Log level gets pinned at a minimum of INFO for file output.
+    CSET.setup_logging(0)
+    assert root_logger.level == logging.INFO
+    # -v
+    CSET.setup_logging(1)
+    assert root_logger.level == logging.INFO
+    # -vv
+    CSET.setup_logging(2)
+    assert root_logger.level == logging.DEBUG
+
+
+def test_main_no_subparser(capsys):
+    """Appropriate error when no subparser is given."""
+    with pytest.raises(SystemExit) as sysexit:
+        CSET.main(["cset"])
+    assert sysexit.value.code == 127
+    assert capsys.readouterr().err == "Please choose a command.\n"
+
+
+def test_main_unhandled_exception_normal(capsys):
+    """Appropriate error when an unhandled exception occurs."""
+    with pytest.raises(SystemExit) as sysexit:
+        CSET.main(
+            [
+                "cset",
+                "bake",
+                "--recipe=/non-existent/recipe.yaml",
+                "--input-dir=/dev/null",
+                "--output-dir=/dev/null",
+            ]
+        )
+    assert sysexit.value.code == 1
+    assert (
+        capsys.readouterr().err
+        == "[Errno 2] No such file or directory: '/non-existent/recipe.yaml'\n"
+    )
 
-    # Gain coverage of __main__.py
-    subprocess.run(["python3", "-m", "CSET", "-h"], check=True)
 
-    # Test verbose options. This is really just to up the coverage number.
-    subprocess.run(["cset", "-v"])
-    subprocess.run(["cset", "-vv"])
+def test_main_unhandled_exception_debug(caplog, capsys):
+    """Appropriate error when an unhandled exception occurs under debug mode."""
+    with pytest.raises(FileNotFoundError):
+        CSET.main(
+            [
+                "cset",
+                "-vv",
+                "bake",
+                "--recipe=/non-existent/recipe.yaml",
+                "--input-dir=/dev/null",
+                "--output-dir=/dev/null",
+            ]
+        )
+    assert (
+        "[Errno 2] No such file or directory: '/non-existent/recipe.yaml'\n"
+        in capsys.readouterr().err
+    )
+    log_record = caplog.records[-1]
+    assert log_record.message == "An unhandled exception occurred."
+    assert log_record.levelno == logging.DEBUG
 
 
-def test_bake_recipe_execution(tmp_path):
+def test_bake_recipe_execution(monkeypatch):
     """Test running CSET recipe from the command line."""
-    subprocess.run(
+    bake_ran = False
+
+    def _bake_test(args, unparsed_args):
+        nonlocal bake_ran
+        bake_ran = True
+        assert args.input_dir == Path("/dev/null")
+        assert args.output_dir == Path("/dev/null")
+        assert args.recipe == Path("tests/test_data/noop_recipe.yaml")
+
+    monkeypatch.setattr(CSET, "_bake_command", _bake_test)
+    CSET.main(
         [
             "cset",
             "bake",
-            f"--input-dir={os.devnull}",
-            f"--output-dir={tmp_path}",
+            "--input-dir=/dev/null",
+            "--output-dir=/dev/null",
             "--recipe=tests/test_data/noop_recipe.yaml",
-        ],
-        check=True,
+        ]
     )
 
 
-def test_bake_invalid_args():
+def test_bake_invalid_args(capsys):
     """Invalid arguments give non-zero exit code."""
-    with pytest.raises(subprocess.CalledProcessError):
-        subprocess.run(
+    with pytest.raises(SystemExit) as sysexit:
+        CSET.main(
             [
                 "cset",
                 "bake",
                 "--recipe=foo",
                 "--input-dir=/tmp",
                 "--output-dir=/tmp",
                 "--not-a-real-option",
-            ],
-            check=True,
+            ]
         )
+    assert sysexit.value.code == 127
+    assert capsys.readouterr().err == "Unknown argument: --not-a-real-option\n"
 
 
-def test_bake_invalid_args_input_dir():
+def test_bake_invalid_args_input_dir(capsys):
     """Missing required input-dir argument for bake."""
-    with pytest.raises(subprocess.CalledProcessError):
-        subprocess.run(
-            ["cset", "bake", "--recipe=foo", "--output-dir=/tmp"], check=True
-        )
+    with pytest.raises(SystemExit) as sysexit:
+        CSET.main(["cset", "bake", "--recipe=foo", "--output-dir=/tmp"])
+    assert sysexit.value.code == 2
+    assert capsys.readouterr().err.endswith(
+        "cset bake: error: the following arguments are required: -i/--input-dir\n"
+    )
 
 
 def test_graph_creation(tmp_path: Path):
@@ -84,116 +174,89 @@ def test_graph_creation(tmp_path: Path):
 
     # Run with output path specified
     output_file = tmp_path / f"{uuid4()}.svg"
-    subprocess.run(
-        (
+    CSET.main(
+        [
             "cset",
             "graph",
             "-o",
             str(output_file),
             "--recipe=tests/test_data/noop_recipe.yaml",
-        ),
-        check=True,
+        ]
     )
     assert output_file.is_file()
     output_file.unlink()
 
 
 def test_graph_details(tmp_path: Path):
-    """Generate a graph with details with details."""
+    """Generate a graph with details."""
     output_file = tmp_path / f"{uuid4()}.svg"
-    subprocess.run(
-        (
+    CSET.main(
+        [
             "cset",
             "graph",
             "--details",
             "-o",
             str(output_file),
             "--recipe=tests/test_data/noop_recipe.yaml",
-        ),
-        check=True,
+        ]
     )
     assert output_file.is_file()
 
 
 def test_cookbook_cwd(tmp_working_dir):
     """Unpacking the recipes into the current working directory."""
-    subprocess.run(["cset", "cookbook", "CAPE_ratio_plot.yaml"], check=True)
+    CSET.main(["cset", "cookbook", "CAPE_ratio_plot.yaml"])
     assert Path("CAPE_ratio_plot.yaml").is_file()
 
 
 def test_cookbook_path(tmp_path: Path):
     """Unpacking the recipes into a specified directory."""
-    subprocess.run(
-        ["cset", "cookbook", "--output-dir", tmp_path, "CAPE_ratio_plot.yaml"],
-        check=True,
+    CSET.main(
+        ["cset", "cookbook", "--output-dir", str(tmp_path), "CAPE_ratio_plot.yaml"]
     )
     assert (tmp_path / "CAPE_ratio_plot.yaml").is_file()
 
 
-def test_cookbook_list_available_recipes():
+def test_cookbook_list_available_recipes(capsys):
     """List all available recipes."""
-    proc = subprocess.run(
-        ["cset", "cookbook", "--details"], capture_output=True, check=True
-    )
+    CSET.main(["cset", "cookbook", "--details"])
+    stdout = capsys.readouterr().out
     # Check start.
-    assert proc.stdout.startswith(b"Available recipes:\n")
+    assert stdout.startswith("Available recipes:\n")
     # Check has some recipes.
-    assert len(proc.stdout.splitlines()) > 3
+    assert len(stdout.splitlines()) > 3
 
 
-def test_cookbook_detail_recipe():
+def test_cookbook_detail_recipe(capsys):
     """Show detail of a recipe."""
-    proc = subprocess.run(
-        [
-            "cset",
-            "cookbook",
-            "--details",
-            "CAPE_ratio_plot.yaml",
-        ],
-        capture_output=True,
-        check=True,
-    )
-    assert proc.stdout.startswith(b"\n\tCAPE_ratio_plot.yaml\n")
+    CSET.main(["cset", "cookbook", "--details", "CAPE_ratio_plot.yaml"])
+    assert capsys.readouterr().out.startswith("\n\tCAPE_ratio_plot.yaml\n")
 
 
 def test_cookbook_non_existent_recipe(tmp_path):
     """Non-existent recipe give non-zero exit code."""
-    with pytest.raises(subprocess.CalledProcessError):
-        subprocess.run(
-            ["cset", "cookbook", "--output-dir", tmp_path, "non-existent.yaml"],
-            check=True,
+    with pytest.raises(SystemExit) as sysexit:
+        CSET.main(
+            ["cset", "cookbook", "--output-dir", str(tmp_path), "non-existent.yaml"]
         )
+    assert sysexit.value.code == 1
 
 
-def test_recipe_id():
+def test_recipe_id(capsys):
     """Get recipe ID for a recipe."""
-    p = subprocess.run(
-        ["cset", "recipe-id", "-r", "tests/test_data/noop_recipe.yaml"],
-        check=True,
-        capture_output=True,
-    )
-    assert p.stdout == b"noop\n"
+    CSET.main(["cset", "recipe-id", "-r", "tests/test_data/noop_recipe.yaml"])
+    assert capsys.readouterr().out == "noop\n"
 
 
-def test_recipe_id_no_title():
+def test_recipe_id_no_title(capsys):
     """Get recipe id for recipe without a title."""
-    p = subprocess.run(
-        ["cset", "recipe-id", "-r", "tests/test_data/ensemble_air_temp.yaml"],
-        check=True,
-        capture_output=True,
-    )
+    CSET.main(["cset", "recipe-id", "-r", "tests/test_data/ensemble_air_temp.yaml"])
     # UUID output + newline.
-    assert len(p.stdout) == 37
+    assert len(capsys.readouterr().out) == 37
 
 
-def test_cset_addopts():
+def test_cset_addopts(capsys, monkeypatch):
     """Lists in CSET_ADDOPTS environment variable don't crash the parser."""
-    environment = dict(os.environ)
-    environment["CSET_ADDOPTS"] = "--LIST='[1, 2, 3]'"
-    p = subprocess.run(
-        ["cset", "recipe-id", "-r", "tests/test_data/addopts_test_recipe.yaml"],
-        check=True,
-        capture_output=True,
-        env=environment,
-    )
-    assert p.stdout == b"list_1_2_3\n"
+    monkeypatch.setenv("CSET_ADDOPTS", "--LIST='[1, 2, 3]'")
+    CSET.main(["cset", "recipe-id", "-r", "tests/test_data/addopts_test_recipe.yaml"])
+    assert capsys.readouterr().out == "list_1_2_3\n"