From 00a5b3a9c8781742122706956ea28f24ca20e029 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 5 Sep 2023 14:35:01 -0500 Subject: [PATCH] Support dbt-cloud config dict in dbt_project.yml (#8527) (#8556) * first pass at adding dbt-cloud config * changelog * fix test, add direct validation --- .../unreleased/Features-20230830-212828.yaml | 6 +++ core/dbt/config/project.py | 6 +++ core/dbt/config/runtime.py | 1 + core/dbt/contracts/project.py | 5 ++ tests/functional/basic/test_project.py | 52 ++++++++++++++++++- 5 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Features-20230830-212828.yaml diff --git a/.changes/unreleased/Features-20230830-212828.yaml b/.changes/unreleased/Features-20230830-212828.yaml new file mode 100644 index 00000000000..935256b9c70 --- /dev/null +++ b/.changes/unreleased/Features-20230830-212828.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Accept a `dbt-cloud` config in dbt_project.yml +time: 2023-08-30T21:28:28.976746-05:00 +custom: + Author: emmyoop + Issue: "8438" diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index f3723e41eb7..1d487d1068a 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -402,6 +402,7 @@ def create_project(self, rendered: RenderComponents) -> "Project": metrics: Dict[str, Any] exposures: Dict[str, Any] vars_value: VarProvider + dbt_cloud: Dict[str, Any] dispatch = cfg.dispatch models = cfg.models @@ -433,6 +434,8 @@ def create_project(self, rendered: RenderComponents) -> "Project": manifest_selectors = SelectorDict.parse_from_selectors_list( rendered.selectors_dict["selectors"] ) + dbt_cloud = cfg.dbt_cloud + project = Project( project_name=name, version=version, @@ -470,6 +473,7 @@ def create_project(self, rendered: RenderComponents) -> "Project": config_version=cfg.config_version, unrendered=unrendered, project_env_vars=project_env_vars, + dbt_cloud=dbt_cloud, ) # sanity check - this means an internal issue project.validate() @@ -572,6 +576,7 @@ class Project: config_version: int unrendered: RenderComponents project_env_vars: Dict[str, Any] + dbt_cloud: Dict[str, Any] @property def all_source_paths(self) -> List[str]: @@ -640,6 +645,7 @@ def to_project_config(self, with_packages=False): "vars": self.vars.to_dict(), "require-dbt-version": [v.to_version_string() for v in self.dbt_version], "config-version": self.config_version, + "dbt-cloud": self.dbt_cloud, } ) if self.query_comment: diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py index d58a9009922..08c16cbcebd 100644 --- a/core/dbt/config/runtime.py +++ b/core/dbt/config/runtime.py @@ -179,6 +179,7 @@ def from_parts( args=args, cli_vars=cli_vars, dependencies=dependencies, + dbt_cloud=project.dbt_cloud, ) # Called by 'load_projects' in this class diff --git a/core/dbt/contracts/project.py b/core/dbt/contracts/project.py index 581932e5888..23a06c7d99e 100644 --- a/core/dbt/contracts/project.py +++ b/core/dbt/contracts/project.py @@ -223,6 +223,7 @@ class Project(HyphenatedDbtClassMixin, Replaceable): ) packages: List[PackageSpec] = field(default_factory=list) query_comment: Optional[Union[QueryComment, NoValue, str]] = field(default_factory=NoValue) + dbt_cloud: Optional[Dict[str, Any]] = None @classmethod def validate(cls, data): @@ -239,6 +240,10 @@ def validate(cls, data): or not isinstance(entry["search_order"], list) ): raise ValidationError(f"Invalid project dispatch config: {entry}") + if "dbt_cloud" in data and not isinstance(data["dbt_cloud"], dict): + raise ValidationError( + f"Invalid dbt_cloud config. Expected a 'dict' but got '{type(data['dbt_cloud'])}'" + ) @dataclass diff --git a/tests/functional/basic/test_project.py b/tests/functional/basic/test_project.py index 10427c5ec3b..080c5d591d0 100644 --- a/tests/functional/basic/test_project.py +++ b/tests/functional/basic/test_project.py @@ -1,5 +1,8 @@ +import os import pytest -from dbt.tests.util import run_dbt, update_config_file +import yaml +from pathlib import Path +from dbt.tests.util import run_dbt, update_config_file, write_config_file from dbt.exceptions import ProjectContractError @@ -62,3 +65,50 @@ def test_invalid_version(self, project): assert "at path ['version']: 'invalid' is not valid under any of the given schemas" in str( excinfo.value ) + + +class TestProjectDbtCloudConfig: + @pytest.fixture(scope="class") + def models(self): + return {"simple_model.sql": simple_model_sql, "simple_model.yml": simple_model_yml} + + def test_dbt_cloud(self, project): + run_dbt(["parse"], expect_pass=True) + conf = yaml.safe_load( + Path(os.path.join(project.project_root, "dbt_project.yml")).read_text() + ) + assert conf == {"name": "test", "profile": "test"} + + config = { + "name": "test", + "profile": "test", + "dbt-cloud": { + "account_id": "123", + "application": "test", + "environment": "test", + "api_key": "test", + }, + } + write_config_file(config, project.project_root, "dbt_project.yml") + run_dbt(["parse"], expect_pass=True) + conf = yaml.safe_load( + Path(os.path.join(project.project_root, "dbt_project.yml")).read_text() + ) + assert conf == config + + +class TestProjectDbtCloudConfigString: + @pytest.fixture(scope="class") + def models(self): + return {"simple_model.sql": simple_model_sql, "simple_model.yml": simple_model_yml} + + def test_dbt_cloud_invalid(self, project): + run_dbt() + config = {"name": "test", "profile": "test", "dbt-cloud": "Some string"} + update_config_file(config, "dbt_project.yml") + expected_err = ( + "at path ['dbt-cloud']: 'Some string' is not valid under any of the given schemas" + ) + with pytest.raises(ProjectContractError) as excinfo: + run_dbt() + assert expected_err in str(excinfo.value)