From 5cfdce3becd173e925519c0c47ab8c14aee4c4f9 Mon Sep 17 00:00:00 2001 From: Andrew Lui Date: Wed, 27 Nov 2024 17:50:39 -0500 Subject: [PATCH 1/4] adds profile_config for Docker --- cosmos/operators/docker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cosmos/operators/docker.py b/cosmos/operators/docker.py index 05671b4d0..81b7196b3 100644 --- a/cosmos/operators/docker.py +++ b/cosmos/operators/docker.py @@ -4,6 +4,7 @@ from airflow.utils.context import Context +from cosmos.config import ProfileConfig from cosmos.operators.base import ( AbstractDbtBaseOperator, DbtBuildMixin, @@ -42,8 +43,10 @@ class DbtDockerBaseOperator(AbstractDbtBaseOperator, DockerOperator): # type: i def __init__( self, image: str, # Make image a required argument since it's required by DockerOperator + profile_config: ProfileConfig | None = None, **kwargs: Any, ) -> None: + self.profile_config = profile_config super().__init__(image=image, **kwargs) def build_and_run_cmd(self, context: Context, cmd_flags: list[str] | None = None) -> Any: From 2b7b82b0075465ff385805663d449128bdce90b2 Mon Sep 17 00:00:00 2001 From: Andrew Lui Date: Wed, 18 Dec 2024 11:15:16 -0500 Subject: [PATCH 2/4] Adds Docker tests back --- tests/test_converter.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_converter.py b/tests/test_converter.py index 9da31f00d..19d9ae4e5 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -134,7 +134,7 @@ def test_validate_arguments_schema_in_task_args(): "execution_mode,operator_args", [ (ExecutionMode.KUBERNETES, {}), - # (ExecutionMode.DOCKER, {"image": "sample-image"}), + (ExecutionMode.DOCKER, {"image": "sample-image"}), ], ) @patch("cosmos.converter.DbtGraph.filtered_nodes", nodes) @@ -168,6 +168,7 @@ def test_converter_creates_dag_with_seed(mock_load_dbt_graph, execution_mode, op "execution_mode,operator_args", [ (ExecutionMode.KUBERNETES, {}), + (ExecutionMode.DOCKER, {"image": "sample-image"}), ], ) @patch("cosmos.converter.DbtGraph.filtered_nodes", nodes) @@ -201,7 +202,7 @@ def test_converter_creates_dag_with_project_path_str(mock_load_dbt_graph, execut "execution_mode,virtualenv_dir,operator_args", [ (ExecutionMode.KUBERNETES, Path("/some/virtualenv/dir"), {}), - # (ExecutionMode.DOCKER, {"image": "sample-image"}), + (ExecutionMode.DOCKER, {"image": "sample-image"}), ], ) @patch("cosmos.converter.DbtGraph.filtered_nodes", nodes) @@ -241,7 +242,7 @@ def test_converter_raises_warning(mock_load_dbt_graph, execution_mode, virtualen "execution_mode,operator_args", [ (ExecutionMode.KUBERNETES, {}), - # (ExecutionMode.DOCKER, {"image": "sample-image"}), + (ExecutionMode.DOCKER, {"image": "sample-image"}), ], ) @patch("cosmos.converter.DbtGraph.filtered_nodes", nodes) @@ -278,7 +279,7 @@ def test_converter_fails_execution_config_no_project_dir(mock_load_dbt_graph, ex "execution_mode,operator_args", [ (ExecutionMode.KUBERNETES, {}), - # (ExecutionMode.DOCKER, {"image": "sample-image"}), + (ExecutionMode.DOCKER, {"image": "sample-image"}), ], ) @patch("cosmos.converter.DbtGraph.filtered_nodes", nodes) @@ -317,7 +318,7 @@ def test_converter_fails_project_config_path_and_execution_config_path( "execution_mode,operator_args", [ (ExecutionMode.KUBERNETES, {}), - # (ExecutionMode.DOCKER, {"image": "sample-image"}), + (ExecutionMode.DOCKER, {"image": "sample-image"}), ], ) @patch("cosmos.converter.DbtGraph.filtered_nodes", nodes) From 3629068ad084094fdb67cef07e984fdbc2fa356a Mon Sep 17 00:00:00 2001 From: Andrew Lui Date: Wed, 18 Dec 2024 11:20:59 -0500 Subject: [PATCH 3/4] Fix virtual env test --- tests/test_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_converter.py b/tests/test_converter.py index 19d9ae4e5..4883605dc 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -202,7 +202,7 @@ def test_converter_creates_dag_with_project_path_str(mock_load_dbt_graph, execut "execution_mode,virtualenv_dir,operator_args", [ (ExecutionMode.KUBERNETES, Path("/some/virtualenv/dir"), {}), - (ExecutionMode.DOCKER, {"image": "sample-image"}), + (ExecutionMode.DOCKER, Path("/some/virtualenv/dir"), {"image": "sample-image"}), ], ) @patch("cosmos.converter.DbtGraph.filtered_nodes", nodes) From 1a1c7d86719fc61803dee17fd30dd7e595dbd3af Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Thu, 19 Dec 2024 17:22:22 +0530 Subject: [PATCH 4/4] Address review comments on PR 1347 --- cosmos/operators/docker.py | 8 ++++++++ docs/getting_started/docker.rst | 12 ++++++++++++ tests/operators/test_docker.py | 27 +++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/cosmos/operators/docker.py b/cosmos/operators/docker.py index 81b7196b3..8dc614cfc 100644 --- a/cosmos/operators/docker.py +++ b/cosmos/operators/docker.py @@ -5,6 +5,7 @@ from airflow.utils.context import Context from cosmos.config import ProfileConfig +from cosmos.exceptions import CosmosValueError from cosmos.operators.base import ( AbstractDbtBaseOperator, DbtBuildMixin, @@ -47,6 +48,13 @@ def __init__( **kwargs: Any, ) -> None: self.profile_config = profile_config + if self.profile_config and not self.profile_config.profiles_yml_filepath: + raise CosmosValueError( + "For ExecutionMode.DOCKER, specifying ProfileConfig only works with profiles_yml_filepath method and " + "it must be specified. ProfileConfig with ProfileMapping method is not supported as the underlying " + "Airflow connections are not available in the Docker container for the mapping to work." + ) + super().__init__(image=image, **kwargs) def build_and_run_cmd(self, context: Context, cmd_flags: list[str] | None = None) -> Any: diff --git a/docs/getting_started/docker.rst b/docs/getting_started/docker.rst index ead5a3d32..000591488 100644 --- a/docs/getting_started/docker.rst +++ b/docs/getting_started/docker.rst @@ -97,3 +97,15 @@ Enable and trigger a run of the `jaffle_shop_docker