From 581a6c0c7b7dca3f392bd26978b0a9af122c6491 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 18 Apr 2024 21:27:50 -0400 Subject: [PATCH] Add a control in that verifies that `dbt-core` functionality is not being used within an adapter (#177) --- .../Under the Hood-20240417-192843.yaml | 6 ++ .pre-commit-config.yaml | 81 ++++++++++++------- dbt/adapters/base/impl.py | 4 +- pyproject.toml | 49 +---------- 4 files changed, 63 insertions(+), 77 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240417-192843.yaml diff --git a/.changes/unreleased/Under the Hood-20240417-192843.yaml b/.changes/unreleased/Under the Hood-20240417-192843.yaml new file mode 100644 index 000000000..94ec9292d --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240417-192843.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Validate that dbt-core and dbt-adapters remain de-coupled +time: 2024-04-17T19:28:43.400023-04:00 +custom: + Author: mikealfare + Issue: "144" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index faf874993..caf342092 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,30 +1,57 @@ default_language_version: - python: python3 + python: python3 repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: check-yaml - args: [--unsafe] - - id: check-json - - id: end-of-file-fixer - - id: trailing-whitespace - - id: check-case-conflict -- repo: local - hooks: - - id: black - name: black - entry: black - language: system - types: [python] - - id: flake8 - name: flake8 - entry: flake8 - language: system - types: [python] - - id: mypy - name: mypy - entry: mypy - language: system - types: [python] +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + args: [--unsafe] + - id: check-json + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-case-conflict + +- repo: https://github.com/dbt-labs/pre-commit-hooks + rev: v0.1.0a1 + hooks: + - id: dbt-core-in-adapters-check + +- repo: https://github.com/psf/black + rev: 24.4.0 + hooks: + - id: black + args: + - --line-length=99 + - --target-version=py38 + - --target-version=py39 + - --target-version=py310 + - --target-version=py311 + - --force-exclude=dbt/adapters/events/adapter_types_pb2.py + +- repo: https://github.com/pycqa/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/ + args: + - --max-line-length=99 + - --select=E,F,W + - --ignore=E203,E501,E704,E741,W503,W504 + - --per-file-ignores=*/__init__.py:F401 + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.9.0 + hooks: + - id: mypy + exclude: dbt/adapters/events/adapter_types_pb2.py|dbt-tests-adapter/dbt/__init__.py + args: + - --explicit-package-bases + - --ignore-missing-imports + - --pretty + - --show-error-codes + files: ^dbt/adapters/ + additional_dependencies: + - types-PyYAML + - types-protobuf + - types-pytz diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index a1df204f8..f58f8aba0 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -1328,8 +1328,8 @@ def calculate_freshness_from_metadata_batch( # Track schema, identifiers of sources for lookup from batch query schema_identifier_to_source = { ( - source.path.get_lowered_part(ComponentName.Schema), - source.path.get_lowered_part(ComponentName.Identifier), + source.path.get_lowered_part(ComponentName.Schema), # type: ignore + source.path.get_lowered_part(ComponentName.Identifier), # type: ignore ): source for source in sources } diff --git a/pyproject.toml b/pyproject.toml index d9c3b4f45..fd72b9927 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,21 +54,11 @@ include = ["dbt/adapters", "dbt/include", "dbt/__init__.py"] [tool.hatch.envs.default] dependencies = [ "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", - "black>=24.3", - "flake8", - "Flake8-pyproject", - "mypy", "pre-commit==3.7.0", - "types-PyYAML", - "types-protobuf", - "types-pytz", ] [tool.hatch.envs.default.scripts] dev = "pre-commit install" -code-quality = "pre-commit run --all-files" # runs others via hooks -black = "python -m black ." -flake8 = "python -m flake8 ." -mypy = "python -m mypy ." +code-quality = "pre-commit run --all-files" [tool.hatch.envs.unit-tests] dependencies = [ @@ -103,45 +93,8 @@ check-sdist = [ ] protobuf = "protoc -I=./dbt/adapters/events --python_out=./dbt/adapters/events ./dbt/adapters/events/adapter_types.proto" -[tool.black] -force-exclude = "dbt/adapters/events/adapter_types_pb2.py" -line-length = 99 -target-version = ['py38'] - -[tool.flake8] -select = ["E", "W", "F"] -ignore = [ - "E203", - "E501", - "E741", - "W503", - "W504", - "E704", # generates false positives on protocols -] -exclude = [ - "dbt/adapters/events/adapter_types_pb2.py", - "tests/functional", - "venv", -] -per-file-ignores = ["*/__init__.py: F401"] - [tool.mypy] -namespace_packages = true -show_error_codes = true -explicit_package_bases = true -ignore_missing_imports = true -pretty = true mypy_path = "third-party-stubs/" -files = [ - "dbt", - "tests/unit", -] -exclude = [ - "dbt/adapters/events/adapter_types_pb2.py", - "dbt-tests-adapter/dbt/__init__.py", # overlaps with `dbt/__init__.py` as expected for namespaces - "venv", - ".hatch_venvs", -] [[tool.mypy.overrides]] module = ["dbt.adapters.events.adapter_types_pb2"] follow_imports = "skip"