Deps lock by justbldwn (#8408)

* ✨ adding installed_packages.json functionality

* ✅ update test_simple_dependency_deps test

* 📝 adding changelog for deps feature via changie

* ✨ restructure deps command, include lock/add

* ✅ add new deps event types to sample_values

* ✅ fix test_simple_dependency_deps test

* 🐛 attempting to fix cli commands

* 🐛 convert dbt deps to dbt deps install

also leave dbt deps as just a new click group

* ✅ update test_command_mutually_exclusive_option

change deps command to deps install

* ✅ update functional tests from deps > deps install

* ✅ change missing deps to deps install

* ✅ convert adapter tests to deps install from deps

* move back to deps and merge more with main

* fix-unittest

* add hash

* foramt yml and update command structure

* nits

* add new param

* nits

* nits

* nits

* fix_tests

* pr_feedback

* nits

* nits

* move_check

* Update Features-20230125-165933.yaml

---------

Co-authored-by: Justin Baldwin <[email protected]>

.changes/unreleased/Features-20230125-165933.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: add log file of installed packages via dbt deps
+time: 2023-01-25T16:59:33.786304-05:00
+custom:
+  Author: jusbaldw ChenyuLInx
+  Issue: "6643"

core/dbt/cli/main.py

@@ -454,13 +454,40 @@ def debug(ctx, **kwargs):
 @p.project_dir
 @p.target
 @p.vars
+@p.source
+@p.dry_run
+@p.lock
+@p.upgrade
+@p.add_package
 @requires.postflight
 @requires.preflight
 @requires.unset_profile
 @requires.project
 def deps(ctx, **kwargs):
-    """Pull the most recent version of the dependencies listed in packages.yml"""
-    task = DepsTask(ctx.obj["flags"], ctx.obj["project"])
+    """Install dbt packages specified.
+    In the following case, a new `package-lock.yml` will be generated and the packages are installed:
+    - user updated the packages.yml
+    - user specify the flag --update, which means for packages that are specified as a
+      range, dbt-core will try to install the newer version
+    Otherwise, deps will use `package-lock.yml` as source of truth to install packages.
+
+    There is a way to add new packages by providing an `--add-package` flag to deps command
+    which will allow user to specify a package they want to add in the format of packagename@version.
+    """
+    flags = ctx.obj["flags"]
+    if flags.ADD_PACKAGE:
+        if not flags.ADD_PACKAGE["version"] and flags.SOURCE != "local":
+            raise BadOptionUsage(
+                message=f"Version is required in --add-package when a package when source is {flags.SOURCE}",
+                option_name="--add-package",
+            )
+    else:
+        if flags.DRY_RUN:
+            raise BadOptionUsage(
+                message="Invalid flag `--dry-run` when not using `--add-package`.",
+                option_name="--dry-run",
+            )
+    task = DepsTask(flags, ctx.obj["project"])
     results = task.run()
     success = task.interpret_results(results)
     return results, success
@@ -555,7 +582,6 @@ def list(ctx, **kwargs):
 def parse(ctx, **kwargs):
     """Parses the project and provides information on performance"""
     # manifest generation and writing happens in @requires.manifest
-
     return ctx.obj["manifest"], True
 
 

core/dbt/cli/option_types.py

@@ -22,6 +22,26 @@ def convert(self, value, param, ctx):
             self.fail(f"String '{value}' is not valid YAML", param, ctx)
 
 
+class Package(ParamType):
+    """The Click STRING type. Converts string into dict with package name and version.
+    Example package:
+        [email protected]
+        package-name
+    """
+
+    name = "NewPackage"
+
+    def convert(self, value, param, ctx):
+        # assume non-string values are a problem
+        if not isinstance(value, str):
+            self.fail(f"Cannot load Package from type {type(value)}", param, ctx)
+        try:
+            package_name, package_version = value.split("@")
+            return {"name": package_name, "version": package_version}
+        except ValueError:
+            return {"name": value, "version": None}
+
+
 class WarnErrorOptionsType(YAML):
     """The Click WarnErrorOptions type. Converts YAML strings into objects."""
 

core/dbt/cli/params.py

@@ -2,10 +2,16 @@
 
 import click
 from dbt.cli.options import MultiOption
-from dbt.cli.option_types import YAML, ChoiceTuple, WarnErrorOptionsType
+from dbt.cli.option_types import YAML, ChoiceTuple, WarnErrorOptionsType, Package
 from dbt.cli.resolvers import default_project_dir, default_profiles_dir
 from dbt.version import get_version_information
 
+add_package = click.option(
+    "--add-package",
+    help="Add a package to current package spec, specify it as package-name@version. Change the source with --source flag.",
+    envvar=None,
+    type=Package(),
+)
 args = click.option(
     "--args",
     envvar=None,
@@ -77,6 +83,14 @@
     hidden=True,
 )
 
+dry_run = click.option(
+    "--dry-run",
+    envvar=None,
+    help="Option to run `dbt deps --add-package` without updating package-lock.yml file.",
+    is_flag=True,
+)
+
+
 enable_legacy_logger = click.option(
     "--enable-legacy-logger/--no-enable-legacy-logger",
     envvar="DBT_ENABLE_LEGACY_LOGGER",
@@ -127,6 +141,13 @@
     default="eager",
 )
 
+lock = click.option(
+    "--lock",
+    envvar=None,
+    help="Generate the package-lock.yml file without install the packages.",
+    is_flag=True,
+)
+
 log_cache_events = click.option(
     "--log-cache-events/--no-log-cache-events",
     help="Enable verbose logging for relational cache events to help when debugging.",
@@ -465,6 +486,14 @@
     is_flag=True,
 )
 
+source = click.option(
+    "--source",
+    envvar=None,
+    help="Source to download page from, must be one of hub, git, or local. Defaults to hub.",
+    type=click.Choice(["hub", "git", "local"], case_sensitive=True),
+    default="hub",
+)
+
 static = click.option(
     "--static",
     help="Generate an additional static_index.html with manifest and catalog built-in.",
@@ -540,6 +569,13 @@
     type=click.Path(),
 )
 
+upgrade = click.option(
+    "--upgrade",
+    envvar=None,
+    help="Upgrade packages to the latest version.",
+    is_flag=True,
+)
+
 debug_connection = click.option(
     "--connection",
     envvar=None,

core/dbt/config/project.py

@@ -16,8 +16,12 @@
 
 from dbt.flags import get_flags
 from dbt import deprecations
-from dbt.constants import DEPENDENCIES_FILE_NAME, PACKAGES_FILE_NAME
-from dbt.clients.system import path_exists, resolve_path_from_base, load_file_contents
+from dbt.constants import (
+    DEPENDENCIES_FILE_NAME,
+    PACKAGES_FILE_NAME,
+    PACKAGE_LOCK_HASH_KEY,
+)
+from dbt.clients.system import path_exists, load_file_contents
 from dbt.clients.yaml_helper import load_yaml_text
 from dbt.contracts.connection import QueryComment
 from dbt.exceptions import (
@@ -94,16 +98,17 @@ def _load_yaml(path):
     return load_yaml_text(contents)
 
 
+def load_yml_dict(file_path):
+    ret = {}
+    if path_exists(file_path):
+        ret = _load_yaml(file_path) or {}
+    return ret
+
+
 def package_and_project_data_from_root(project_root):
-    package_filepath = resolve_path_from_base(PACKAGES_FILE_NAME, project_root)
-    dependencies_filepath = resolve_path_from_base(DEPENDENCIES_FILE_NAME, project_root)
 
-    packages_yml_dict = {}
-    dependencies_yml_dict = {}
-    if path_exists(package_filepath):
-        packages_yml_dict = _load_yaml(package_filepath) or {}
-    if path_exists(dependencies_filepath):
-        dependencies_yml_dict = _load_yaml(dependencies_filepath) or {}
+    packages_yml_dict = load_yml_dict(f"{project_root}/{PACKAGES_FILE_NAME}")
+    dependencies_yml_dict = load_yml_dict(f"{project_root}/{DEPENDENCIES_FILE_NAME}")
 
     if "packages" in packages_yml_dict and "packages" in dependencies_yml_dict:
         msg = "The 'packages' key cannot be specified in both packages.yml and dependencies.yml"
@@ -127,6 +132,8 @@ def package_config_from_data(packages_data: Dict[str, Any]) -> PackageConfig:
     if not packages_data:
         packages_data = {"packages": []}
 
+    if PACKAGE_LOCK_HASH_KEY in packages_data:
+        packages_data.pop(PACKAGE_LOCK_HASH_KEY)
     try:
         PackageConfig.validate(packages_data)
         packages = PackageConfig.from_dict(packages_data)
@@ -548,6 +555,7 @@ def from_project_root(
             packages_specified_path,
         ) = package_and_project_data_from_root(project_root)
         selectors_dict = selector_data_from_root(project_root)
+
         return cls.from_dicts(
             project_root=project_root,
             project_dict=project_dict,

core/dbt/constants.py

@@ -12,6 +12,8 @@
 DBT_PROJECT_FILE_NAME = "dbt_project.yml"
 PACKAGES_FILE_NAME = "packages.yml"
 DEPENDENCIES_FILE_NAME = "dependencies.yml"
+PACKAGE_LOCK_FILE_NAME = "package-lock.yml"
 MANIFEST_FILE_NAME = "manifest.json"
 SEMANTIC_MANIFEST_FILE_NAME = "semantic_manifest.json"
 PARTIAL_PARSE_FILE_NAME = "partial_parse.msgpack"
+PACKAGE_LOCK_HASH_KEY = "sha1_hash"

core/dbt/deps/resolver.py

@@ -135,3 +135,15 @@ def resolve_packages(
     resolved = final.resolved()
     _check_for_duplicate_project_names(resolved, project, renderer)
     return resolved
+
+
+def resolve_lock_packages(packages: List[PackageContract]) -> List[PinnedPackage]:
+    lock_packages = PackageListing.from_contracts(packages)
+    final = PackageListing()
+
+    for package in lock_packages:
+        final.incorporate(package)
+
+    resolved = final.resolved()
+
+    return resolved

core/dbt/events/types.proto

@@ -1586,6 +1586,48 @@ message NoNodesForSelectionCriteriaMsg {
     NoNodesForSelectionCriteria data = 2;
 }
 
+// M031
+message DepsLockUpdating{
+    string lock_filepath = 1;
+}
+
+message DepsLockUpdatingMsg{
+    EventInfo info = 1;
+    DepsLockUpdating data = 2;
+}
+
+// M032
+message DepsAddPackage{
+    string package_name = 1;
+    string version = 2;
+    string packages_filepath = 3;
+}
+
+message DepsAddPackageMsg{
+    EventInfo info = 1;
+    DepsAddPackage data = 2;
+}
+
+//M033
+message DepsFoundDuplicatePackage{
+    map<string, string> removed_package = 1;
+}
+
+message DepsFoundDuplicatePackageMsg{
+    EventInfo info = 1;
+    DepsFoundDuplicatePackage data = 2;
+}
+
+//M034
+message DepsVersionMissing{
+    string source = 1;
+}
+
+message DepsVersionMissingMsg{
+    EventInfo info = 1;
+    DepsVersionMissing data = 2;
+}
+
 // Q - Node execution
 
 // Q001

core/dbt/events/types.py

@@ -1504,6 +1504,30 @@ def message(self) -> str:
         return f"The selection criterion '{self.spec_raw}' does not match any nodes"
 
 
+class DepsLockUpdating(InfoLevel):
+    def code(self):
+        return "M031"
+
+    def message(self) -> str:
+        return f"Updating lock file in file path: {self.lock_filepath}"
+
+
+class DepsAddPackage(InfoLevel):
+    def code(self):
+        return "M032"
+
+    def message(self) -> str:
+        return f"Added new package {self.package_name}@{self.version} to {self.packages_filepath}"
+
+
+class DepsFoundDuplicatePackage(InfoLevel):
+    def code(self):
+        return "M033"
+
+    def message(self) -> str:
+        return f"Found duplicate package in packages.yml, removing: {self.removed_package}"
+
+
 # =======================================================
 # Q - Node execution
 # =======================================================