From 452d31192b809f035f24f0432592c1acee715534 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 11 Oct 2023 13:46:11 -0400 Subject: [PATCH] [Backport 1.1.latest] respect project root when loading seeds (#8762) (#8815) --- .../unreleased/Fixes-20231006-134551.yaml | 6 ++ core/dbt/context/providers.py | 13 ++- core/dbt/tests/util.py | 4 + tests/functional/partial_parsing/fixtures.py | 66 ++++++++++++++ .../partial_parsing/test_partial_parsing.py | 86 +++++++++++++++++++ tests/functional/utils.py | 14 +++ 6 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20231006-134551.yaml create mode 100644 tests/functional/partial_parsing/fixtures.py create mode 100644 tests/functional/partial_parsing/test_partial_parsing.py create mode 100644 tests/functional/utils.py diff --git a/.changes/unreleased/Fixes-20231006-134551.yaml b/.changes/unreleased/Fixes-20231006-134551.yaml new file mode 100644 index 00000000000..9dea4f6e194 --- /dev/null +++ b/.changes/unreleased/Fixes-20231006-134551.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Enable seeds to be handled from stored manifest data +time: 2023-10-06T13:45:51.925546-04:00 +custom: + Author: michelleark + Issue: "6875" diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index f98e8622be0..ab8e052eec8 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -724,7 +724,18 @@ def load_agate_table(self) -> agate.Table: raise_compiler_error( "can only load_agate_table for seeds (got a {})".format(self.model.resource_type) ) - path = os.path.join(self.model.root_path, self.model.original_file_path) + + # include package_path for seeds defined in packages + package_path = ( + os.path.join(self.config.packages_install_path, self.model.package_name) + if self.model.package_name != self.config.project_name + else "." + ) + path = os.path.join(self.config.project_root, package_path, self.model.original_file_path) + if not os.path.exists(path): + assert self.model.root_path + path = os.path.join(self.model.root_path, self.model.original_file_path) + column_types = self.model.config.column_types try: table = agate_helper.from_csv(path, text_columns=column_types) diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index 078faeb36ef..92d30b88ca8 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -127,6 +127,10 @@ def read_file(*paths): return contents +def rename_dir(src_directory_path, dest_directory_path): + os.rename(src_directory_path, dest_directory_path) + + # Get an artifact (usually from the target directory) such as # manifest.json or catalog.json to use in a test def get_artifact(*paths): diff --git a/tests/functional/partial_parsing/fixtures.py b/tests/functional/partial_parsing/fixtures.py new file mode 100644 index 00000000000..6938b0bfcd7 --- /dev/null +++ b/tests/functional/partial_parsing/fixtures.py @@ -0,0 +1,66 @@ +local_dependency__dbt_project_yml = """ + +name: 'local_dep' +version: '1.0' +config-version: 2 + +profile: 'default' + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] + +require-dbt-version: '>=0.1.0' + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + + +seeds: + quote_columns: False + +""" + +local_dependency__models__schema_yml = """ +version: 2 +sources: + - name: seed_source + schema: "{{ var('schema_override', target.schema) }}" + tables: + - name: "seed" + columns: + - name: id + tests: + - unique + +""" + +local_dependency__models__model_to_import_sql = """ +select * from {{ ref('seed') }} + +""" + +local_dependency__macros__dep_macro_sql = """ +{% macro some_overridden_macro() -%} +100 +{%- endmacro %} + +""" + +local_dependency__seeds__seed_csv = """id +1 +""" + +model_one_sql = """ +select 1 as fun + +""" + +model_two_sql = """ +select 1 as notfun + +""" diff --git a/tests/functional/partial_parsing/test_partial_parsing.py b/tests/functional/partial_parsing/test_partial_parsing.py new file mode 100644 index 00000000000..62a6553f5cf --- /dev/null +++ b/tests/functional/partial_parsing/test_partial_parsing.py @@ -0,0 +1,86 @@ +from argparse import Namespace +import pytest + +import dbt.flags as flags +from dbt.tests.util import ( + run_dbt, + write_file, + rename_dir, +) +from tests.functional.utils import up_one +from dbt.tests.fixtures.project import write_project_files +from tests.functional.partial_parsing.fixtures import ( + model_one_sql, + model_two_sql, + local_dependency__dbt_project_yml, + local_dependency__models__schema_yml, + local_dependency__models__model_to_import_sql, + local_dependency__macros__dep_macro_sql, + local_dependency__seeds__seed_csv, +) + +import os + +os.environ["DBT_PP_TEST"] = "true" + + +class TestPortablePartialParsing: + @pytest.fixture(scope="class") + def models(self): + return { + "model_one.sql": model_one_sql, + } + + @pytest.fixture(scope="class") + def packages(self): + return {"packages": [{"local": "local_dependency"}]} + + @pytest.fixture(scope="class") + def local_dependency_files(self): + return { + "dbt_project.yml": local_dependency__dbt_project_yml, + "models": { + "schema.yml": local_dependency__models__schema_yml, + "model_to_import.sql": local_dependency__models__model_to_import_sql, + }, + "macros": {"dep_macro.sql": local_dependency__macros__dep_macro_sql}, + "seeds": {"seed.csv": local_dependency__seeds__seed_csv}, + } + + def rename_project_root(self, project, new_project_root): + with up_one(new_project_root): + rename_dir(project.project_root, new_project_root) + project.project_root = new_project_root + # flags.project_dir is set during the project test fixture, and is persisted across run_dbt calls, + # so it needs to be reset between invocations + flags.set_from_args(Namespace(PROJECT_DIR=new_project_root), None) + + @pytest.fixture(scope="class", autouse=True) + def initial_run_and_rename_project_dir(self, project, local_dependency_files): + initial_project_root = project.project_root + renamed_project_root = os.path.join(project.project_root.dirname, "renamed_project_dir") + + write_project_files(project.project_root, "local_dependency", local_dependency_files) + + # initial run + run_dbt(["deps"]) + assert len(run_dbt(["seed"])) == 1 + assert len(run_dbt(["run"])) == 2 + + self.rename_project_root(project, renamed_project_root) + yield + self.rename_project_root(project, initial_project_root) + + def test_pp_renamed_project_dir_unchanged_project_contents(self, project): + # partial parse same project in new absolute dir location, using partial_parse.msgpack created in previous dir + run_dbt(["deps"]) + assert len(run_dbt(["--partial-parse", "seed"])) == 1 + assert len(run_dbt(["--partial-parse", "run"])) == 2 + + def test_pp_renamed_project_dir_changed_project_contents(self, project): + write_file(model_two_sql, project.project_root, "models", "model_two.sql") + + # partial parse changed project in new absolute dir location, using partial_parse.msgpack created in previous dir + run_dbt(["deps"]) + len(run_dbt(["--partial-parse", "seed"])) == 1 + len(run_dbt(["--partial-parse", "run"])) == 3 diff --git a/tests/functional/utils.py b/tests/functional/utils.py new file mode 100644 index 00000000000..ddfe367856b --- /dev/null +++ b/tests/functional/utils.py @@ -0,0 +1,14 @@ +import os +from contextlib import contextmanager +from typing import Optional +from pathlib import Path + + +@contextmanager +def up_one(return_path: Optional[Path] = None): + current_path = Path.cwd() + os.chdir("../") + try: + yield + finally: + os.chdir(return_path or current_path)