From 4a3e12bc9a0b815f67ba32e42ff1c0e4dcb4ac8c Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Tue, 17 Dec 2024 12:31:20 +0000 Subject: [PATCH] Improve dbt command execution logs to troubleshoot `None` values (#1392) Recently, an Astronomer customer reported they were facing the following error while trying to run Cosmos and couldn't troubleshoot it (the screenshot was a courtesy by the customer, when using Cosmos 1.7.1): ![type-error](https://github.com/user-attachments/assets/bfb15103-d6f8-425c-ae0e-25b40953d3d7) The following log line was sub-optimal since it errored if the value was None: https://github.com/astronomer/astronomer-cosmos/blob/43998d246661b0f27580cf9f314d19aeef785b6a/cosmos/dbt/graph.py#L136 To change just the log line was not enough since the `None` would also break the subprocess command execution (screenshot also by the customer when trying 1.8.0a1): ![type-error-2](https://github.com/user-attachments/assets/d8980b4e-eed1-4948-825f-3f219e84b1ee) By changing both, the source problem became evident to the customer (screenshot by the customer when trying 1.8.0a2): ![unable-to-run](https://github.com/user-attachments/assets/80906ee0-8c17-4615-8ca2-b87a704db5e1) --- cosmos/dbt/graph.py | 1 + tests/dbt/test_graph.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index be37ec298..6c207f2ca 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -133,6 +133,7 @@ def is_freshness_effective(freshness: Optional[dict[str, Any]]) -> bool: def run_command(command: list[str], tmp_dir: Path, env_vars: dict[str, str]) -> str: """Run a command in a subprocess, returning the stdout.""" + command = [str(arg) if arg is not None else "" for arg in command] logger.info("Running command: `%s`", " ".join(command)) logger.debug("Environment variable keys: %s", env_vars.keys()) process = Popen( diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 59d71869d..499cc219c 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -1121,6 +1121,20 @@ def test_run_command(mock_popen, stdout, returncode): assert return_value == stdout +@patch("cosmos.dbt.graph.Popen") +def test_run_command_none_argument(mock_popen, caplog): + fake_command = ["invalid-cmd", None] + fake_dir = Path("fake_dir") + env_vars = {"fake": "env_var"} + + mock_popen.return_value.communicate.return_value = ("Invalid None argument", None) + with pytest.raises(CosmosLoadDbtException) as exc_info: + run_command(fake_command, fake_dir, env_vars) + + expected = "Unable to run ['invalid-cmd', ''] due to the error:\nInvalid None argument" + assert str(exc_info.value) == expected + + def test_parse_dbt_ls_output_real_life_customer_bug(caplog): dbt_ls_output = """ 11:20:43 Running with dbt=1.7.6