diff --git a/scripts/build-dist.sh b/scripts/build-dist.sh deleted file mode 100755 index 5dd0fa17f22..00000000000 --- a/scripts/build-dist.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -DBT_PATH="$( cd "$(dirname "$0")/.." ; pwd -P )" - -PYTHON_BIN=${PYTHON_BIN:-python} - -echo "$PYTHON_BIN" - -set -x - -rm -rf "$DBT_PATH"/dist -rm -rf "$DBT_PATH"/build -mkdir -p "$DBT_PATH"/dist - -rm -rf "$DBT_PATH"/core/dist -rm -rf "$DBT_PATH"core/build -cd "$DBT_PATH"/core -$PYTHON_BIN setup.py sdist bdist_wheel -cp -r "$DBT_PATH"/"core"/dist/* "$DBT_PATH"/dist/ - - -set +x diff --git a/scripts/check_libyaml.py b/scripts/check_libyaml.py deleted file mode 100755 index 276d6517bc5..00000000000 --- a/scripts/check_libyaml.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python -try: - from yaml import CLoader as Loader, CSafeLoader as SafeLoader, CDumper as Dumper # noqa: F401 -except ImportError: - from yaml import Loader, SafeLoader, Dumper # noqa: F401 - -if Loader.__name__ == "CLoader": - print("libyaml is working") -elif Loader.__name__ == "Loader": - print("libyaml is not working") - print("Check the python executable and pyyaml for libyaml support") diff --git a/scripts/collect-artifact-schema.py b/scripts/collect-artifact-schema.py deleted file mode 100755 index 5bedbfbbd6b..00000000000 --- a/scripts/collect-artifact-schema.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python -from dataclasses import dataclass -from argparse import ArgumentParser -from pathlib import Path -import json -from typing import Type, Dict, Any - -from dbt.contracts.graph.manifest import WritableManifest -from dbt.artifacts.schemas.catalog import CatalogArtifact -from dbt.artifacts.schemas.run import RunResultsArtifact -from dbt.artifacts.schemas.freshness import FreshnessExecutionResultArtifact -from dbt.artifacts.schemas.base import VersionedSchema -from dbt_common.clients.system import write_file - - -@dataclass -class ArtifactInfo: - path: str - name: str - json_schema: Dict[str, Any] - - @classmethod - def from_artifact_cls( - cls, - artifact_cls: Type[VersionedSchema], - ) -> "ArtifactInfo": - return cls( - path=artifact_cls.dbt_schema_version.path, - name=artifact_cls.dbt_schema_version.name, - json_schema=artifact_cls.json_schema(), - ) - - def write_schema(self, dest_dir: Path): - write_file(str(dest_dir / self.path), json.dumps(self.json_schema, indent=2)) - - -@dataclass -class Arguments: - artifact: str - path: Path - - @classmethod - def parse(cls) -> "Arguments": - parser = ArgumentParser(prog="Collect and write dbt arfifact schema") - parser.add_argument( - "--path", - type=Path, - help="The dir to write artifact schema", - ) - - parser.add_argument( - "--artifact", - type=str, - choices=["manifest", "sources", "run-results", "catalog"], - help="The name of the artifact to update", - ) - - parsed = parser.parse_args() - return cls(artifact=parsed.artifact, path=parsed.path) - - -def collect_artifact_schema(args: Arguments): - artifacts = [ - FreshnessExecutionResultArtifact, - RunResultsArtifact, - CatalogArtifact, - # WritableManifest introduces new definitions in hologram which are likely - # getting persisted across invocations of json_schema and making their - # way to other written artifacts - so write it last as a short-term fix. - # https://github.com/dbt-labs/dbt-core/issues/7604 - WritableManifest, - ] - filtered_artifacts = filter( - lambda a: a.dbt_schema_version.name == args.artifact or args.artifact is None, artifacts - ) - artifact_infos = [] - for artifact_cls in filtered_artifacts: - artifact_infos.append(ArtifactInfo.from_artifact_cls(artifact_cls)) - - if args and args.path is not None: - for artifact_info in artifact_infos: - dest_dir = args.path.resolve() - artifact_info.write_schema(dest_dir) - else: - artifacts_dict = { - artifact_info.name: artifact_info.json_schema for artifact_info in artifact_infos - } - print(json.dumps(artifacts_dict)) - - -def main(): - parsed = Arguments.parse() - collect_artifact_schema(parsed) - - -if __name__ == "__main__": - main() diff --git a/scripts/collect-dbt-contexts.py b/scripts/collect-dbt-contexts.py deleted file mode 100644 index 9e93494c9b1..00000000000 --- a/scripts/collect-dbt-contexts.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python - -import inspect -import json -from dataclasses import dataclass -from typing import List, Optional, Iterable, Union, Dict, Any -from dbt_common.dataclass_schema import dbtClassMixin - - -from dbt.context.base import BaseContext -from dbt.context.target import TargetContext -from dbt.context.providers import ModelContext, MacroContext - - -CONTEXTS_MAP = { - "base": BaseContext, - "target": TargetContext, - "model": ModelContext, - "macro": MacroContext, -} - - -@dataclass -class ContextValue(dbtClassMixin): - name: str - value: str # a type description - doc: Optional[str] - - -@dataclass -class MethodArgument(dbtClassMixin): - name: str - value: str # a type description - - -@dataclass -class ContextMethod(dbtClassMixin): - name: str - args: List[MethodArgument] - result: str # a type description - doc: Optional[str] - - -@dataclass -class Unknown(dbtClassMixin): - name: str - value: str - doc: Optional[str] - - -ContextMember = Union[ContextValue, ContextMethod, Unknown] - - -def _get_args(func: inspect.Signature) -> Iterable[MethodArgument]: - found_first = False - for argname, arg in func.parameters.items(): - if found_first is False and argname in {"self", "cls"}: - continue - if found_first is False: - found_first = True - - yield MethodArgument( - name=argname, - value=inspect.formatannotation(arg.annotation), - ) - - -def collect(cls): - values = [] - for name, v in cls._context_members_.items(): - attrname = cls._context_attrs_[name] - attrdef = getattr(cls, attrname) - doc = getattr(attrdef, "__doc__") - if inspect.isfunction(attrdef): - sig = inspect.signature(attrdef) - result = inspect.formatannotation(sig.return_annotation) - sig_good_part = ContextMethod( - name=name, - args=list(_get_args(sig)), - result=result, - doc=doc, - ) - elif isinstance(attrdef, property): - sig = inspect.signature(attrdef.fget) - sig_txt = inspect.formatannotation(sig.return_annotation) - sig_good_part = ContextValue(name=name, value=sig_txt, doc=doc) - else: - sig_good_part = Unknown(name=name, value=repr(attrdef), doc=doc) - values.append(sig_good_part) - - return values - - -@dataclass -class ContextCatalog(dbtClassMixin): - base: List[ContextMember] - target: List[ContextMember] - model: List[ContextMember] - macro: List[ContextMember] - schema: Dict[str, Any] - - -def main(): - catalog = ContextCatalog( - base=collect(BaseContext), - target=collect(TargetContext), - model=collect(ModelContext), - macro=collect(MacroContext), - schema=ContextCatalog.json_schema(), - ) - print(json.dumps(catalog.to_dict())) - - -if __name__ == "__main__": - main() diff --git a/scripts/env-setup.sh b/scripts/env-setup.sh deleted file mode 100644 index 42968b79eb1..00000000000 --- a/scripts/env-setup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# Set environment variables required for integration tests -echo "DBT_INVOCATION_ENV=github-actions" >> $GITHUB_ENV -echo "DBT_TEST_USER_1=dbt_test_user_1" >> $GITHUB_ENV -echo "DBT_TEST_USER_2=dbt_test_user_2" >> $GITHUB_ENV -echo "DBT_TEST_USER_3=dbt_test_user_3" >> $GITHUB_ENV diff --git a/scripts/migrate-adapters.py b/scripts/migrate-adapters.py deleted file mode 100644 index ead59bd271e..00000000000 --- a/scripts/migrate-adapters.py +++ /dev/null @@ -1,60 +0,0 @@ -import argparse -import ast -from collections import namedtuple -from pathlib import Path - -Import = namedtuple("Import", ["module", "name", "alias"]) - - -def get_imports(path): - with open(path) as fh: - root = ast.parse(fh.read(), path) - - for node in ast.iter_child_nodes(root): - if isinstance(node, ast.Import): - module = [] - elif isinstance(node, ast.ImportFrom): - module = node.module.split(".") - else: - continue - - for n in node.names: - yield Import(module, n.name.split("."), n.asname) - - -parser = argparse.ArgumentParser("migrate_adapters") -parser.add_argument("path", help="The path to run the migration tool over.", type=str) -args = parser.parse_args() - -path = Path(args.path) -pathlist = path.rglob("*.py") - -total_dbt_imports = 0 -invalid_dbt_imports = 0 -path_to_invalid_imports = {} -for path in pathlist: - path_to_invalid_imports[path] = [] - for imported_module in get_imports(str(path)): - if imported_module.module and imported_module.module[0] == "dbt": - total_dbt_imports += 1 - if imported_module.module[1] not in ("common", "adapters"): - invalid_dbt_imports += 1 - path_to_invalid_imports[path].append( - f"{'.'.join(imported_module.module)}::{imported_module.name[0]}" - ) - -migrated_imports = total_dbt_imports - invalid_dbt_imports -migrated_imports_progress = round((migrated_imports / total_dbt_imports) * 100, 2) - -for path, invalid_imports in path_to_invalid_imports.items(): - if invalid_imports: - print() - print(f"\033[92m{path}:\033[0m") - for invalid_import in invalid_imports: - print(f" - {invalid_import}") - -print() -print( - f"migration progress: {migrated_imports_progress}% of dbt imports are valid (from adapters or common)" -) -print(f"remaining core imports: {invalid_dbt_imports}") diff --git a/scripts/update_dev_packages.sh b/scripts/update_dev_packages.sh deleted file mode 100755 index 3f140f04b89..00000000000 --- a/scripts/update_dev_packages.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -e -set -e - -repo=$1 -ref=$2 -target_req_file="dev-requirements.txt" - -req_sed_pattern="s|${repo}.git@main|${repo}.git@${ref}|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$req_sed_pattern" $target_req_file -else - sed -i "$req_sed_pattern" $target_req_file -fi diff --git a/tests/functional/unit_testing/test_state.py b/tests/functional/unit_testing/test_state.py index 4df874726d0..7235a5411d7 100644 --- a/tests/functional/unit_testing/test_state.py +++ b/tests/functional/unit_testing/test_state.py @@ -136,7 +136,6 @@ def test_unit_test_defer_state(self, project): assert sorted([r.status for r in results]) == ["fail", "pass", "pass", "pass"] - class TestUnitTestDeferDoesntOverwrite(UnitTestState): @pytest.fixture(scope="class") def models(self):