Fix lock for git subduer (#9019) (#9028)

* wip

* add tests

* changelog

* nits

* pr feedback

* nits

(cherry picked from commit 01d481b)

Co-authored-by: Chenyu Li <[email protected]>

.changes/unreleased/Fixes-20231107-092358.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix git repository with subdirectory for Deps
+time: 2023-11-07T09:23:58.214271-08:00
+custom:
+  Author: ChenyuLInx
+  Issue: "9000"

core/dbt/deps/base.py

@@ -4,7 +4,7 @@
 import tempfile
 from contextlib import contextmanager
 from pathlib import Path
-from typing import List, Optional, Generic, TypeVar
+from typing import List, Optional, Generic, TypeVar, Dict
 
 from dbt.clients import system
 from dbt.contracts.project import ProjectPackageMetadata
@@ -84,6 +84,10 @@ def install(self, project, renderer):
     def nice_version_name(self):
         raise NotImplementedError
 
+    @abc.abstractmethod
+    def to_dict(self) -> Dict[str, str]:
+        raise NotImplementedError
+
     def fetch_metadata(self, project, renderer):
         if not self._cached_metadata:
             self._cached_metadata = self._fetch_metadata(project, renderer)

core/dbt/deps/git.py

@@ -1,5 +1,5 @@
 import os
-from typing import List, Optional
+from typing import List, Optional, Dict
 
 from dbt.clients import git, system
 from dbt.config.project import PartialProject, Project
@@ -51,6 +51,15 @@ def __init__(
         self.subdirectory = subdirectory
         self._checkout_name = md5sum(self.name)
 
+    def to_dict(self) -> Dict[str, str]:
+        ret = {
+            "git": self.git,
+            "revision": self.revision,
+        }
+        if self.subdirectory:
+            ret["subdirectory"] = self.subdirectory
+        return ret
+
     def get_version(self):
         return self.revision
 

core/dbt/deps/local.py

@@ -1,4 +1,5 @@
 import shutil
+from typing import Dict
 
 from dbt.clients import system
 from dbt.deps.base import PinnedPackage, UnpinnedPackage
@@ -29,6 +30,11 @@ class LocalPinnedPackage(LocalPackageMixin, PinnedPackage):
     def __init__(self, local: str) -> None:
         super().__init__(local)
 
+    def to_dict(self) -> Dict[str, str]:
+        return {
+            "local": self.local,
+        }
+
     def get_version(self):
         return None
 

core/dbt/deps/registry.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, Dict
 
 from dbt import semver
 from dbt.flags import get_flags
@@ -40,6 +40,12 @@ def __init__(self, package: str, version: str, version_latest: str) -> None:
     def name(self):
         return self.package
 
+    def to_dict(self) -> Dict[str, str]:
+        return {
+            "package": self.package,
+            "version": self.version,
+        }
+
     def source_type(self):
         return "hub"
 

core/dbt/deps/tarball.py

@@ -1,3 +1,5 @@
+from typing import Dict
+
 from dbt.contracts.project import RegistryPackageMetadata, TarballPackage
 from dbt.deps.base import PinnedPackage, UnpinnedPackage
 
@@ -26,6 +28,13 @@ def __init__(self, tarball: str, package: str) -> None:
     def name(self):
         return self.package
 
+    def to_dict(self) -> Dict[str, str]:
+        return {
+            "tarball": self.tarball,
+            "version": self.version,
+            "package": self.package,
+        }
+
     def get_version(self):
         return self.version
 

core/dbt/task/deps.py

@@ -75,11 +75,11 @@ def _create_packages_yml_entry(package: str, version: Optional[str], source: str
     if source == "hub":
         package_key = "package"
 
+    packages_yml_entry = {package_key: package}
+
     if source == "git":
         version_key = "revision"
 
-    packages_yml_entry = {package_key: package}
-
     if version:
         if "," in version:
             version = version.split(",")  # type: ignore
@@ -192,10 +192,7 @@ def lock(self) -> None:
         # this loop is to create the package-lock.yml in the same format as original packages.yml
         # package-lock.yml includes both the stated packages in packages.yml along with dependent packages
         for package in resolved_deps:
-            lock_entry = _create_packages_yml_entry(
-                package.name, package.get_version(), package.source_type()
-            )
-            packages_installed["packages"].append(lock_entry)
+            packages_installed["packages"].append(package.to_dict())
         packages_installed[PACKAGE_LOCK_HASH_KEY] = _create_sha1_hash(
             self.project.packages.packages
         )

tests/functional/dependencies/test_simple_dependency.py

@@ -216,6 +216,43 @@ def test_simple_dependency_deps(self, project):
         run_dbt(["deps"])
 
 
+class TestSimpleDependencyWithSubdirs(object):
+    # dbt should convert these into a single dependency internally
+    @pytest.fixture(scope="class")
+    def packages(self):
+        return {
+            "packages": [
+                {
+                    "git": "https://github.com/dbt-labs/dbt-multipe-packages.git",
+                    "subdirectory": "dbt-utils-main",
+                    "revision": "v0.1.0",
+                },
+                {
+                    "git": "https://github.com/dbt-labs/dbt-multipe-packages.git",
+                    "subdirectory": "dbt-date-main",
+                    "revision": "v0.1.0",
+                },
+            ]
+        }
+
+    def test_git_with_multiple_subdir(self, project):
+        run_dbt(["deps"])
+        assert os.path.exists("package-lock.yml")
+        expected = """packages:
+- git: https://github.com/dbt-labs/dbt-multipe-packages.git
+  revision: v0.1.0
+  subdirectory: dbt-utils-main
+- git: https://github.com/dbt-labs/dbt-multipe-packages.git
+  revision: v0.1.0
+  subdirectory: dbt-date-main
+sha1_hash: 0b643b06246ca34be82ef09524a30635d37aa3be
+"""
+        with open("package-lock.yml") as fp:
+            contents = fp.read()
+        assert contents == expected
+        assert len(os.listdir("dbt_packages")) == 2
+
+
 class TestRekeyedDependencyWithSubduplicates(object):
     # this revision of dbt-integration-project requires [email protected], which the
     # package config handling should detect