[Backport 1.4.latest] respect project root when loading seeds (#8762) (…

…#8806)

.changes/unreleased/Fixes-20231006-134551.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Enable seeds to be handled from stored manifest data
+time: 2023-10-06T13:45:51.925546-04:00
+custom:
+  Author: michelleark
+  Issue: "6875"

core/dbt/context/providers.py

@@ -788,8 +788,18 @@ def try_or_compiler_error(
     def load_agate_table(self) -> agate.Table:
         if not isinstance(self.model, SeedNode):
             raise LoadAgateTableNotSeedError(self.model.resource_type, node=self.model)
-        assert self.model.root_path
-        path = os.path.join(self.model.root_path, self.model.original_file_path)
+
+        # include package_path for seeds defined in packages
+        package_path = (
+            os.path.join(self.config.packages_install_path, self.model.package_name)
+            if self.model.package_name != self.config.project_name
+            else "."
+        )
+        path = os.path.join(self.config.project_root, package_path, self.model.original_file_path)
+        if not os.path.exists(path):
+            assert self.model.root_path
+            path = os.path.join(self.model.root_path, self.model.original_file_path)
+
         column_types = self.model.config.column_types
         try:
             table = agate_helper.from_csv(path, text_columns=column_types)

core/dbt/tests/util.py

@@ -169,6 +169,10 @@ def rm_dir(directory_path):
         raise FileNotFoundError(f"{directory_path} does not exist.")
 
 
+def rename_dir(src_directory_path, dest_directory_path):
+    os.rename(src_directory_path, dest_directory_path)
+
+
 # Get an artifact (usually from the target directory) such as
 # manifest.json or catalog.json to use in a test
 def get_artifact(*paths):

tests/functional/partial_parsing/fixtures.py

@@ -0,0 +1,66 @@
+local_dependency__dbt_project_yml = """
+
+name: 'local_dep'
+version: '1.0'
+config-version: 2
+
+profile: 'default'
+
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+
+require-dbt-version: '>=0.1.0'
+
+target-path: "target"  # directory which will store compiled SQL files
+clean-targets:         # directories to be removed by `dbt clean`
+    - "target"
+    - "dbt_packages"
+
+
+seeds:
+  quote_columns: False
+
+"""
+
+local_dependency__models__schema_yml = """
+version: 2
+sources:
+  - name: seed_source
+    schema: "{{ var('schema_override', target.schema) }}"
+    tables:
+      - name: "seed"
+        columns:
+          - name: id
+            tests:
+              - unique
+
+"""
+
+local_dependency__models__model_to_import_sql = """
+select * from {{ ref('seed') }}
+
+"""
+
+local_dependency__macros__dep_macro_sql = """
+{% macro some_overridden_macro() -%}
+100
+{%- endmacro %}
+
+"""
+
+local_dependency__seeds__seed_csv = """id
+1
+"""
+
+model_one_sql = """
+select 1 as fun
+
+"""
+
+model_two_sql = """
+select 1 as notfun
+
+"""

tests/functional/partial_parsing/test_partial_parsing.py

@@ -0,0 +1,86 @@
+from argparse import Namespace
+import pytest
+
+import dbt.flags as flags
+from dbt.tests.util import (
+    run_dbt,
+    write_file,
+    rename_dir,
+)
+from tests.functional.utils import up_one
+from dbt.tests.fixtures.project import write_project_files
+from tests.functional.partial_parsing.fixtures import (
+    model_one_sql,
+    model_two_sql,
+    local_dependency__dbt_project_yml,
+    local_dependency__models__schema_yml,
+    local_dependency__models__model_to_import_sql,
+    local_dependency__macros__dep_macro_sql,
+    local_dependency__seeds__seed_csv,
+)
+
+import os
+
+os.environ["DBT_PP_TEST"] = "true"
+
+
+class TestPortablePartialParsing:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "model_one.sql": model_one_sql,
+        }
+
+    @pytest.fixture(scope="class")
+    def packages(self):
+        return {"packages": [{"local": "local_dependency"}]}
+
+    @pytest.fixture(scope="class")
+    def local_dependency_files(self):
+        return {
+            "dbt_project.yml": local_dependency__dbt_project_yml,
+            "models": {
+                "schema.yml": local_dependency__models__schema_yml,
+                "model_to_import.sql": local_dependency__models__model_to_import_sql,
+            },
+            "macros": {"dep_macro.sql": local_dependency__macros__dep_macro_sql},
+            "seeds": {"seed.csv": local_dependency__seeds__seed_csv},
+        }
+
+    def rename_project_root(self, project, new_project_root):
+        with up_one(new_project_root):
+            rename_dir(project.project_root, new_project_root)
+            project.project_root = new_project_root
+            # flags.project_dir is set during the project test fixture, and is persisted across run_dbt calls,
+            # so it needs to be reset between invocations
+            flags.set_from_args(Namespace(PROJECT_DIR=new_project_root), None)
+
+    @pytest.fixture(scope="class", autouse=True)
+    def initial_run_and_rename_project_dir(self, project, local_dependency_files):
+        initial_project_root = project.project_root
+        renamed_project_root = os.path.join(project.project_root.dirname, "renamed_project_dir")
+
+        write_project_files(project.project_root, "local_dependency", local_dependency_files)
+
+        # initial run
+        run_dbt(["deps"])
+        assert len(run_dbt(["seed"])) == 1
+        assert len(run_dbt(["run"])) == 2
+
+        self.rename_project_root(project, renamed_project_root)
+        yield
+        self.rename_project_root(project, initial_project_root)
+
+    def test_pp_renamed_project_dir_unchanged_project_contents(self, project):
+        # partial parse same project in new absolute dir location, using partial_parse.msgpack created in previous dir
+        run_dbt(["deps"])
+        assert len(run_dbt(["--partial-parse", "seed"])) == 1
+        assert len(run_dbt(["--partial-parse", "run"])) == 2
+
+    def test_pp_renamed_project_dir_changed_project_contents(self, project):
+        write_file(model_two_sql, project.project_root, "models", "model_two.sql")
+
+        # partial parse changed project in new absolute dir location, using partial_parse.msgpack created in previous dir
+        run_dbt(["deps"])
+        len(run_dbt(["--partial-parse", "seed"])) == 1
+        len(run_dbt(["--partial-parse", "run"])) == 3

tests/functional/utils.py

@@ -0,0 +1,14 @@
+import os
+from contextlib import contextmanager
+from typing import Optional
+from pathlib import Path
+
+
+@contextmanager
+def up_one(return_path: Optional[Path] = None):
+    current_path = Path.cwd()
+    os.chdir("../")
+    try:
+        yield
+    finally:
+        os.chdir(return_path or current_path)