From 8825ccb8bf2c1343f91a5f3c6ea0c7a05376f1ee Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:16:35 -0400 Subject: [PATCH 1/7] Add query tag back to incremental models (#1212) * add query tag back to incremental models * fix query tag test --- .../macros/materializations/incremental.sql | 1 + tests/functional/query_tag/test_query_tags.py | 34 +++++++------------ 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/dbt/include/snowflake/macros/materializations/incremental.sql b/dbt/include/snowflake/macros/materializations/incremental.sql index 12645cd11..d73525d6d 100644 --- a/dbt/include/snowflake/macros/materializations/incremental.sql +++ b/dbt/include/snowflake/macros/materializations/incremental.sql @@ -58,6 +58,7 @@ {% materialization incremental, adapter='snowflake', supported_languages=['sql', 'python'] -%} + {% set original_query_tag = set_query_tag() %} {#-- Set vars --#} {%- set full_refresh_mode = (should_full_refresh()) -%} diff --git a/tests/functional/query_tag/test_query_tags.py b/tests/functional/query_tag/test_query_tags.py index 421aae1b6..4ddafabb2 100644 --- a/tests/functional/query_tag/test_query_tags.py +++ b/tests/functional/query_tag/test_query_tags.py @@ -1,6 +1,7 @@ import pytest from dbt.tests.util import run_dbt + snapshots__snapshot_query_tag_sql = """ {% snapshot snapshot_query_tag %} {{ @@ -14,16 +15,15 @@ }} select 1 as id, 'blue' as color {% endsnapshot %} - """ + models__table_model_query_tag_sql = """ {{ config(materialized = 'table') }} - select 1 as id - """ + models__models_config_yml = """ version: 2 @@ -33,23 +33,21 @@ - name: id data_tests: - unique - """ + models__view_model_query_tag_sql = """ {{ config(materialized = 'view') }} - select 1 as id - """ + models__incremental_model_query_tag_sql = """ {{ config(materialized = 'incremental', unique_key = 'id') }} - select 1 as id - """ + macros__check_tag_sql = """ {% macro check_query_tag() %} @@ -61,12 +59,12 @@ {% endif %} {% endmacro %} - """ + seeds__seed_query_tag_csv = """id 1 -""" +""".strip() class TestQueryTag: @@ -95,20 +93,14 @@ def seeds(self): def project_config_update(self, prefix): return { "config-version": 2, - "models": { - "tests": {"query_tag": prefix, "post-hook": "{{ check_tag() }}"}, - }, - "seeds": { - "tests": {"query_tag": prefix, "post-hook": "{{ check_tag() }}"}, - }, - "snapshots": { - "tests": {"query_tag": prefix, "post-hook": "{{ check_tag() }}"}, - }, + "models": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}, + "seeds": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}, + "snapshots": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}, "tests": {"test": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}}, } def build_all_with_query_tags(self, project, prefix): - run_dbt(["build", "--vars", '{{"check_tag": "{}"}}'.format(prefix)]) + run_dbt(["build", "--vars", '{{"query_tag": "{}"}}'.format(prefix)]) def test_snowflake_query_tag(self, project, prefix): self.build_all_with_query_tags(project, prefix) @@ -130,7 +122,7 @@ def profiles_config_update(self, prefix): return {"query_tag": prefix} def build_all_with_query_tags(self, project, prefix): - run_dbt(["build", "--vars", '{{"check_tag": "{}"}}'.format(prefix)]) + run_dbt(["build", "--vars", '{{"query_tag": "{}"}}'.format(prefix)]) def test_snowflake_query_tag(self, project, prefix): self.build_all_with_query_tags(project, prefix) From 78f86674bc726000686a484c4d2f0e2f9d350ed1 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:48:05 -0400 Subject: [PATCH 2/7] Drop support for Python 3.8 (#1211) * drop support for python 3.8 --- .../Breaking Changes-20241016-183143.yaml | 6 +++ .github/scripts/integration-test-matrix.js | 4 +- .github/workflows/main.yml | 8 +-- .pre-commit-config.yaml | 1 - CONTRIBUTING.md | 2 +- Makefile | 10 ++-- dev-requirements.txt | 3 +- docker/Dockerfile | 4 +- docker/dev.Dockerfile | 52 +++++++++---------- setup.py | 8 ++- tox.ini | 6 +-- 11 files changed, 53 insertions(+), 51 deletions(-) create mode 100644 .changes/unreleased/Breaking Changes-20241016-183143.yaml diff --git a/.changes/unreleased/Breaking Changes-20241016-183143.yaml b/.changes/unreleased/Breaking Changes-20241016-183143.yaml new file mode 100644 index 000000000..26cc4b6de --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20241016-183143.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: Drop support for Python 3.8 +time: 2024-10-16T18:31:43.4167-04:00 +custom: + Author: mikealfare + Issue: "1211" diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 81386c54e..756c21d5e 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -1,6 +1,6 @@ module.exports = ({ context }) => { - const defaultPythonVersion = "3.8"; - const supportedPythonVersions = ["3.8", "3.9", "3.10", "3.11", "3.12"]; + const defaultPythonVersion = "3.9"; + const supportedPythonVersions = ["3.9", "3.10", "3.11", "3.12"]; const supportedAdapters = ["snowflake"]; // if PR, generate matrix based on files changed and PR labels diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 95ce18033..4913917f4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,7 +50,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -71,7 +71,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] env: TOXENV: "unit" @@ -127,7 +127,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -175,7 +175,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-12, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] dist-type: ['whl', 'gz'] steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e14455e28..807e32a9a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,6 @@ repos: - id: black args: - --line-length=99 - - --target-version=py38 - --target-version=py39 - --target-version=py310 - --target-version=py311 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 45e0054c5..5b68aa03a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,7 +79,7 @@ WARNING: The parameters in your `test.env` file must link to a valid Snowflake a There are a few methods for running tests locally. #### `tox` -`tox` automatically runs unit tests against several Python versions using its own virtualenvs. Run `tox -p` to run unit tests for Python 3.8, Python 3.9, Python 3.10, and `flake8` in parallel. Run `tox -e py38` to invoke tests on Python version 3.8 only (use py38, py39, or py310). Tox recipes are found in `tox.ini`. +`tox` automatically runs unit tests against several Python versions using its own virtualenvs. Run `tox -p` to run unit tests for Python 3.9 and Python 3.10, and `flake8` in parallel. Run `tox -e py39` to invoke tests on Python version 3.9 only (use py39 or py310). Tox recipes are found in `tox.ini`. #### `pytest` You may run a specific test or group of tests using `pytest` directly. Activate a Python virtualenv active with dev dependencies installed. Then, run tests like so: diff --git a/Makefile b/Makefile index b42de9147..9a2376f47 100644 --- a/Makefile +++ b/Makefile @@ -12,20 +12,20 @@ dev-uninstall: ## Uninstalls all packages while maintaining the virtual environm pip uninstall -y dbt-snowflake .PHONY: unit -unit: ## Runs unit tests with py38. +unit: ## Runs unit tests with py39. @\ - tox -e py38 + tox -e py39 .PHONY: test -test: ## Runs unit tests with py38 and code checks against staged changes. +test: ## Runs unit tests with py39 and code checks against staged changes. @\ - tox -p -e py38; \ + tox -p -e py39; \ pre-commit run --all-files .PHONY: integration integration: ## Runs snowflake integration tests with py38. @\ - tox -e py38-snowflake -- + tox -e py39-snowflake -- .PHONY: clean @echo "cleaning repo" diff --git a/dev-requirements.txt b/dev-requirements.txt index f3d120eec..906003768 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,8 +6,7 @@ git+https://github.com/dbt-labs/dbt-common.git # dev ipdb~=0.13.13 -pre-commit~=3.7.0;python_version>="3.9" -pre-commit~=3.5.0;python_version<"3.9" +pre-commit~=3.7.0 # test ddtrace==2.3.0 diff --git a/docker/Dockerfile b/docker/Dockerfile index d256dcac4..17315b12d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # this image gets published to GHCR for production use ARG py_version=3.11.2 -FROM python:$py_version-slim-bullseye as base +FROM python:$py_version-slim-bullseye AS base RUN apt-get update \ && apt-get dist-upgrade -y \ @@ -25,7 +25,7 @@ ENV LANG=C.UTF-8 RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir -FROM base as dbt-snowflake +FROM base AS dbt-snowflake ARG commit_ref=main diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile index 0fc667048..44f86f005 100644 --- a/docker/dev.Dockerfile +++ b/docker/dev.Dockerfile @@ -1,43 +1,43 @@ # this image does not get published, it is intended for local development only, see `Makefile` for usage -FROM ubuntu:22.04 as base +FROM ubuntu:24.04 AS base # prevent python installation from asking for time zone region ARG DEBIAN_FRONTEND=noninteractive # add python repository RUN apt-get update \ - && apt-get install -y software-properties-common=0.99.22.9 \ - && add-apt-repository -y ppa:deadsnakes/ppa \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* + && apt-get install -y software-properties-common=0.99.48 \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* # install python RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential=12.9ubuntu3 \ - git-all=1:2.34.1-1ubuntu1.10 \ - python3.8=3.8.19-1+jammy1 \ - python3.8-dev=3.8.19-1+jammy1 \ - python3.8-distutils=3.8.19-1+jammy1 \ - python3.8-venv=3.8.19-1+jammy1 \ - python3-pip=22.0.2+dfsg-1ubuntu0.4 \ - python3-wheel=0.37.1-2ubuntu0.22.04.1 \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* + && apt-get install -y --no-install-recommends \ + build-essential=12.10ubuntu1 \ + git-all=1:2.43.0-1ubuntu7.1 \ + python3.9=3.9.20-1+noble1 \ + python3.9-dev=3.9.20-1+noble1 \ + python3.9-distutils=3.9.20-1+noble1 \ + python3.9-venv=3.9.20-1+noble1 \ + python3-pip=24.0+dfsg-1ubuntu1 \ + python3-wheel=0.42.0-2 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ # update the default system interpreter to the newly installed version -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 -FROM base as dbt-snowflake-dev +FROM base AS dbt-snowflake-dev -HEALTHCHECK CMD python3 --version || exit 1 +HEALTHCHECK CMD python --version || exit 1 # send stdout/stderr to terminal ENV PYTHONUNBUFFERED=1 @@ -47,4 +47,4 @@ WORKDIR /opt/code VOLUME /opt/code # create a virtual environment -RUN python3 -m venv /opt/venv +RUN python -m venv /opt/venv diff --git a/setup.py b/setup.py index f542b9fcb..c0716341d 100644 --- a/setup.py +++ b/setup.py @@ -5,9 +5,9 @@ import sys # require python 3.8 or newer -if sys.version_info < (3, 8): +if sys.version_info < (3, 9): print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.8 or higher.") + print("Please upgrade to Python 3.9 or higher.") sys.exit(1) @@ -72,12 +72,10 @@ def _plugin_version() -> str: "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ], - python_requires=">=3.8", + python_requires=">=3.9", ) diff --git a/tox.ini b/tox.ini index d6f040a61..f6952efaf 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] skipsdist = True -envlist = py38,py39,py310,py311,py312 +envlist = py39,py310,py311,py312 -[testenv:{unit,py38,py39,py310,py311,py312,py}] +[testenv:{unit,py39,py310,py311,py312,py}] description = unit testing skip_install = true passenv = @@ -13,7 +13,7 @@ deps = -rdev-requirements.txt -e. -[testenv:{integration,py38,py39,py310,py311,py312,py}-{snowflake}] +[testenv:{integration,py39,py310,py311,py312,py}-{snowflake}] description = adapter plugin integration testing skip_install = true passenv = From ea33cbec035c32052008c07c90ef799e024aa7dc Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:08:10 -0700 Subject: [PATCH 3/7] Add url. (#1222) --- dbt/include/snowflake/macros/relations/table/create.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/snowflake/macros/relations/table/create.sql b/dbt/include/snowflake/macros/relations/table/create.sql index e60b93039..e2141df4d 100644 --- a/dbt/include/snowflake/macros/relations/table/create.sql +++ b/dbt/include/snowflake/macros/relations/table/create.sql @@ -1,7 +1,7 @@ {% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%} {%- if relation.is_iceberg_format and not adapter.behavior.enable_iceberg_materializations.no_warn %} - {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to .') %} + {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to https://docs.getdbt.com/reference/resource-configs/snowflake-configs.') %} {%- endif %} {%- set materialization_prefix = relation.get_ddl_prefix_for_create(config.model.config, temporary) -%} From df643a0ed4bec6e0e961594d8171eb2c808d249b Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:20:45 -0700 Subject: [PATCH 4/7] Add snowflake telemetry. (#1209) * Add snowflake telemetry. * Add changelog. * Temporary dev branch switch. * Correct version import * bump ci * Temporary dev branch switch. * Temporary dev branch switch take 2. * Alter to meet new base schema changes. * Fix input args. * Change field name. --------- Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .../Under the Hood-20241016-035544.yaml | 6 +++++ dbt/adapters/snowflake/impl.py | 17 ++++++++++++ dev-requirements.txt | 2 +- setup.py | 2 +- tests/unit/test_adapter_telemetry.py | 27 +++++++++++++++++++ 5 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20241016-035544.yaml create mode 100644 tests/unit/test_adapter_telemetry.py diff --git a/.changes/unreleased/Under the Hood-20241016-035544.yaml b/.changes/unreleased/Under the Hood-20241016-035544.yaml new file mode 100644 index 000000000..59e4f70de --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241016-035544.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add telemetry function +time: 2024-10-16T03:55:44.144174-07:00 +custom: + Author: versusfacit + Issue: "301" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 6320893e1..89c21f531 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -4,6 +4,7 @@ from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability +from dbt.adapters.contracts.relation import RelationConfig from dbt.adapters.sql import SQLAdapter from dbt.adapters.sql.impl import ( LIST_SCHEMAS_MACRO_NAME, @@ -25,6 +26,7 @@ SnowflakeRelationType, TableFormat, ) + from dbt.adapters.snowflake import SnowflakeColumn from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation @@ -419,3 +421,18 @@ def valid_incremental_strategies(self): def debug_query(self): """Override for DebugTask method""" self.execute("select 1 as id") + + @classmethod + def _get_adapter_specific_run_info(cls, config: RelationConfig) -> Dict[str, Any]: + table_format: Optional[str] = None + if ( + config + and hasattr(config, "_extra") + and (relation_format := config._extra.get("table_format")) + ): + table_format = relation_format + + return { + "adapter_type": "snowflake", + "table_format": table_format, + } diff --git a/dev-requirements.txt b/dev-requirements.txt index 906003768..0ceb365db 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ # install latest changes in dbt-core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-adapters.git +git+https://github.com/dbt-labs/dbt-adapters.git@ADAP-301/add-adapter-telemetry git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git diff --git a/setup.py b/setup.py index c0716341d..fcea43309 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def _plugin_version() -> str: include_package_data=True, install_requires=[ "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.7,<2.0", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@ADAP-301/add-adapter-telemetry", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", diff --git a/tests/unit/test_adapter_telemetry.py b/tests/unit/test_adapter_telemetry.py new file mode 100644 index 000000000..498676b77 --- /dev/null +++ b/tests/unit/test_adapter_telemetry.py @@ -0,0 +1,27 @@ +from unittest import mock + +import dbt.adapters.snowflake.__version__ + +from dbt.adapters.snowflake.impl import SnowflakeAdapter +from dbt.adapters.base.relation import AdapterTrackingRelationInfo + + +def test_telemetry_with_snowflake_details(): + mock_model_config = mock.MagicMock() + mock_model_config._extra = mock.MagicMock() + mock_model_config._extra = { + "adapter_type": "snowflake", + "table_format": "iceberg", + } + + res = SnowflakeAdapter.get_adapter_run_info(mock_model_config) + + assert res.adapter_name == "snowflake" + assert res.base_adapter_version == dbt.adapters.__about__.version + assert res.adapter_version == dbt.adapters.snowflake.__version__.version + assert res.model_adapter_details == { + "adapter_type": "snowflake", + "table_format": "iceberg", + } + + assert type(res) is AdapterTrackingRelationInfo From ad4ccadb0411826bd3053800431bbc7cfc9a78c2 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 29 Oct 2024 07:50:49 -0700 Subject: [PATCH 5/7] Revert requirements. (#1225) Co-authored-by: VersusFacit --- dev-requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 0ceb365db..906003768 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ # install latest changes in dbt-core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-adapters.git@ADAP-301/add-adapter-telemetry +git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git diff --git a/setup.py b/setup.py index fcea43309..c0716341d 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def _plugin_version() -> str: include_package_data=True, install_requires=[ "dbt-common>=1.10,<2.0", - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@ADAP-301/add-adapter-telemetry", + "dbt-adapters>=1.7,<2.0", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", From 1d299923e34c96f2e96a5215ac196658f86ce1d1 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:35:37 -0400 Subject: [PATCH 6/7] Add test cases for generic tests config (#1172) * add test cases for generic tests config * update tests --------- Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Co-authored-by: Colin --- .../functional/generic_test_tests/__init__.py | 0 tests/functional/generic_test_tests/_files.py | 90 +++++++++++++++++++ .../functional/generic_test_tests/_models.py | 0 .../functional/generic_test_tests/_schemas.py | 0 .../generic_test_tests/test_generic_tests.py | 54 +++++++++++ 5 files changed, 144 insertions(+) create mode 100644 tests/functional/generic_test_tests/__init__.py create mode 100644 tests/functional/generic_test_tests/_files.py create mode 100644 tests/functional/generic_test_tests/_models.py create mode 100644 tests/functional/generic_test_tests/_schemas.py create mode 100644 tests/functional/generic_test_tests/test_generic_tests.py diff --git a/tests/functional/generic_test_tests/__init__.py b/tests/functional/generic_test_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/generic_test_tests/_files.py b/tests/functional/generic_test_tests/_files.py new file mode 100644 index 000000000..a9743e43e --- /dev/null +++ b/tests/functional/generic_test_tests/_files.py @@ -0,0 +1,90 @@ +SCHEMA__CONTROL = """ +version: 2 +models: + - name: colors + columns: + - name: color + data_tests: + - not_null +""" + + +SCHEMA__EXPLICIT_WAREHOUSE = """ +version: 2 +models: + - name: colors + columns: + - name: color + data_tests: + - not_null: + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SCHEMA__NOT_NULL = """ +version: 2 +models: + - name: facts + columns: + - name: value + data_tests: + - not_null: + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SCHEMA__RELATIONSHIPS = """ +version: 2 +models: + - name: facts + columns: + - name: color + data_tests: + - relationships: + to: ref('my_colors') + field: color + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SCHEMA__ACCEPTED_VALUES = """ +version: 2 +models: + - name: colors + columns: + - name: color + data_tests: + - accepted_values: + values: ['blue', 'red', 'green'] + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SEED__COLORS = """ +color +blue +green +red +yellow +""".strip() + + +# record 10 is missing a value +# record 7 has a color that's not on COLORS +SEED__FACTS = """ +id,color,value +1,blue,10 +2,red,20 +3,green,30 +4,yellow,40 +5,blue,50 +6,red,60 +7,orange,70 +8,green,80 +9,yellow,90 +10,blue, +""".strip() diff --git a/tests/functional/generic_test_tests/_models.py b/tests/functional/generic_test_tests/_models.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/generic_test_tests/_schemas.py b/tests/functional/generic_test_tests/_schemas.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/generic_test_tests/test_generic_tests.py b/tests/functional/generic_test_tests/test_generic_tests.py new file mode 100644 index 000000000..a653a363b --- /dev/null +++ b/tests/functional/generic_test_tests/test_generic_tests.py @@ -0,0 +1,54 @@ +import pytest + +from dbt.tests.util import run_dbt, run_dbt_and_capture + +from tests.functional.generic_test_tests import _files + + +class TestWarehouseConfig: + + @pytest.fixture(scope="class") + def seeds(self): + return { + "colors.csv": _files.SEED__COLORS, + "facts.csv": _files.SEED__FACTS, + } + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + yield + + +class TestWarehouseConfigControl(TestWarehouseConfig): + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": _files.SCHEMA__CONTROL} + + def test_expected_warehouse(self, project): + results, logs = run_dbt_and_capture(["test"]) + assert len(results) == 1 + + +class TestWarehouseConfigExplicitWarehouse(TestWarehouseConfig): + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": _files.SCHEMA__EXPLICIT_WAREHOUSE} + + def test_expected_warehouse(self, project): + _, logs = run_dbt_and_capture(["test", "--log-level", "debug"]) + assert "use warehouse " in logs + + +class TestWarehouseConfigNotNull(TestWarehouseConfig): + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": _files.SCHEMA__NOT_NULL} + + def test_expected_warehouse(self, project): + _, logs = run_dbt_and_capture(["test", "--log-level", "debug"], expect_pass=False) + assert "use warehouse " in logs From a16654bed4d67d1a922bf87e579a285c35bd55ac Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:57:52 -0800 Subject: [PATCH 7/7] move github runner from macos-12 to macos-14 (#1230) * move github runner from macos-12 to macos-14 --- .github/scripts/integration-test-matrix.js | 4 ++-- .github/workflows/main.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 756c21d5e..e2c88b00b 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -44,7 +44,7 @@ module.exports = ({ context }) => { if (labels.includes("test macos") || testAllLabel) { include.push({ - os: "macos-12", + os: "macos-14", adapter, "python-version": pythonVersion, }); @@ -78,7 +78,7 @@ module.exports = ({ context }) => { // additionally include runs for all adapters, on macos and windows, // but only for the default python version for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-12"]) { + for (const operatingSystem of ["windows-latest", "macos-14"]) { include.push({ os: operatingSystem, adapter: adapter, diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4913917f4..24d2fa60b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -174,7 +174,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-12, windows-latest] + os: [ubuntu-latest, macos-14, windows-latest] python-version: ['3.9', '3.10', '3.11', '3.12'] dist-type: ['whl', 'gz']