From 7d60d6361e979977da08ccdd42c28b2b87e1e535 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 14:30:50 -0500 Subject: [PATCH] Fix lock for git subduer (#9019) (#9028) * wip * add tests * changelog * nits * pr feedback * nits (cherry picked from commit 01d481bc8d7326ee5b49c0c2f679b09b87d9ba9f) Co-authored-by: Chenyu Li --- .../unreleased/Fixes-20231107-092358.yaml | 6 +++ core/dbt/deps/base.py | 6 ++- core/dbt/deps/git.py | 11 +++++- core/dbt/deps/local.py | 6 +++ core/dbt/deps/registry.py | 8 +++- core/dbt/deps/tarball.py | 9 +++++ core/dbt/task/deps.py | 9 ++--- .../dependencies/test_simple_dependency.py | 37 +++++++++++++++++++ 8 files changed, 83 insertions(+), 9 deletions(-) create mode 100644 .changes/unreleased/Fixes-20231107-092358.yaml diff --git a/.changes/unreleased/Fixes-20231107-092358.yaml b/.changes/unreleased/Fixes-20231107-092358.yaml new file mode 100644 index 00000000000..24b947a993f --- /dev/null +++ b/.changes/unreleased/Fixes-20231107-092358.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix git repository with subdirectory for Deps +time: 2023-11-07T09:23:58.214271-08:00 +custom: + Author: ChenyuLInx + Issue: "9000" diff --git a/core/dbt/deps/base.py b/core/dbt/deps/base.py index f72878422aa..43f07866cbe 100644 --- a/core/dbt/deps/base.py +++ b/core/dbt/deps/base.py @@ -4,7 +4,7 @@ import tempfile from contextlib import contextmanager from pathlib import Path -from typing import List, Optional, Generic, TypeVar +from typing import List, Optional, Generic, TypeVar, Dict from dbt.clients import system from dbt.contracts.project import ProjectPackageMetadata @@ -84,6 +84,10 @@ def install(self, project, renderer): def nice_version_name(self): raise NotImplementedError + @abc.abstractmethod + def to_dict(self) -> Dict[str, str]: + raise NotImplementedError + def fetch_metadata(self, project, renderer): if not self._cached_metadata: self._cached_metadata = self._fetch_metadata(project, renderer) diff --git a/core/dbt/deps/git.py b/core/dbt/deps/git.py index cc3fa1af6a2..4d0a02ff6e9 100644 --- a/core/dbt/deps/git.py +++ b/core/dbt/deps/git.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional +from typing import List, Optional, Dict from dbt.clients import git, system from dbt.config.project import PartialProject, Project @@ -51,6 +51,15 @@ def __init__( self.subdirectory = subdirectory self._checkout_name = md5sum(self.name) + def to_dict(self) -> Dict[str, str]: + ret = { + "git": self.git, + "revision": self.revision, + } + if self.subdirectory: + ret["subdirectory"] = self.subdirectory + return ret + def get_version(self): return self.revision diff --git a/core/dbt/deps/local.py b/core/dbt/deps/local.py index 18c9af01de3..282e2ed206f 100644 --- a/core/dbt/deps/local.py +++ b/core/dbt/deps/local.py @@ -1,4 +1,5 @@ import shutil +from typing import Dict from dbt.clients import system from dbt.deps.base import PinnedPackage, UnpinnedPackage @@ -29,6 +30,11 @@ class LocalPinnedPackage(LocalPackageMixin, PinnedPackage): def __init__(self, local: str) -> None: super().__init__(local) + def to_dict(self) -> Dict[str, str]: + return { + "local": self.local, + } + def get_version(self): return None diff --git a/core/dbt/deps/registry.py b/core/dbt/deps/registry.py index 351a9985206..14489254407 100644 --- a/core/dbt/deps/registry.py +++ b/core/dbt/deps/registry.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Dict from dbt import semver from dbt.flags import get_flags @@ -40,6 +40,12 @@ def __init__(self, package: str, version: str, version_latest: str) -> None: def name(self): return self.package + def to_dict(self) -> Dict[str, str]: + return { + "package": self.package, + "version": self.version, + } + def source_type(self): return "hub" diff --git a/core/dbt/deps/tarball.py b/core/dbt/deps/tarball.py index 16c9cb0a20d..09462158dbe 100644 --- a/core/dbt/deps/tarball.py +++ b/core/dbt/deps/tarball.py @@ -1,3 +1,5 @@ +from typing import Dict + from dbt.contracts.project import RegistryPackageMetadata, TarballPackage from dbt.deps.base import PinnedPackage, UnpinnedPackage @@ -26,6 +28,13 @@ def __init__(self, tarball: str, package: str) -> None: def name(self): return self.package + def to_dict(self) -> Dict[str, str]: + return { + "tarball": self.tarball, + "version": self.version, + "package": self.package, + } + def get_version(self): return self.version diff --git a/core/dbt/task/deps.py b/core/dbt/task/deps.py index 1433c9a48ca..22a338d2524 100644 --- a/core/dbt/task/deps.py +++ b/core/dbt/task/deps.py @@ -75,11 +75,11 @@ def _create_packages_yml_entry(package: str, version: Optional[str], source: str if source == "hub": package_key = "package" + packages_yml_entry = {package_key: package} + if source == "git": version_key = "revision" - packages_yml_entry = {package_key: package} - if version: if "," in version: version = version.split(",") # type: ignore @@ -192,10 +192,7 @@ def lock(self) -> None: # this loop is to create the package-lock.yml in the same format as original packages.yml # package-lock.yml includes both the stated packages in packages.yml along with dependent packages for package in resolved_deps: - lock_entry = _create_packages_yml_entry( - package.name, package.get_version(), package.source_type() - ) - packages_installed["packages"].append(lock_entry) + packages_installed["packages"].append(package.to_dict()) packages_installed[PACKAGE_LOCK_HASH_KEY] = _create_sha1_hash( self.project.packages.packages ) diff --git a/tests/functional/dependencies/test_simple_dependency.py b/tests/functional/dependencies/test_simple_dependency.py index f9b4f58658c..b8c17cfc0e6 100644 --- a/tests/functional/dependencies/test_simple_dependency.py +++ b/tests/functional/dependencies/test_simple_dependency.py @@ -216,6 +216,43 @@ def test_simple_dependency_deps(self, project): run_dbt(["deps"]) +class TestSimpleDependencyWithSubdirs(object): + # dbt should convert these into a single dependency internally + @pytest.fixture(scope="class") + def packages(self): + return { + "packages": [ + { + "git": "https://github.com/dbt-labs/dbt-multipe-packages.git", + "subdirectory": "dbt-utils-main", + "revision": "v0.1.0", + }, + { + "git": "https://github.com/dbt-labs/dbt-multipe-packages.git", + "subdirectory": "dbt-date-main", + "revision": "v0.1.0", + }, + ] + } + + def test_git_with_multiple_subdir(self, project): + run_dbt(["deps"]) + assert os.path.exists("package-lock.yml") + expected = """packages: +- git: https://github.com/dbt-labs/dbt-multipe-packages.git + revision: v0.1.0 + subdirectory: dbt-utils-main +- git: https://github.com/dbt-labs/dbt-multipe-packages.git + revision: v0.1.0 + subdirectory: dbt-date-main +sha1_hash: 0b643b06246ca34be82ef09524a30635d37aa3be +""" + with open("package-lock.yml") as fp: + contents = fp.read() + assert contents == expected + assert len(os.listdir("dbt_packages")) == 2 + + class TestRekeyedDependencyWithSubduplicates(object): # this revision of dbt-integration-project requires dbt-utils.git@0.5.0, which the # package config handling should detect