From 32c8547ae42169f8147c5fbec2712c22b7f9db33 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 12 Apr 2024 19:19:05 -0400 Subject: [PATCH 1/5] Install `psycopg2` based on platform (#60) --- .../unreleased/Fixes-20240412-153154.yaml | 6 + .github/workflows/integration-tests.yml | 30 +- hatch_build.py | 56 - pyproject.toml | 17 +- tests/functional/partial_parsing/fixtures.py | 1228 ----------------- .../partial_parsing/test_file_diff.py | 63 - .../partial_parsing/test_partial_parsing.py | 824 ----------- .../test_pp_disabled_config.py | 224 --- .../partial_parsing/test_pp_docs.py | 257 ---- .../partial_parsing/test_pp_groups.py | 155 --- .../partial_parsing/test_pp_metrics.py | 85 -- .../partial_parsing/test_pp_vars.py | 398 ------ .../partial_parsing/test_versioned_models.py | 128 -- 13 files changed, 47 insertions(+), 3424 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240412-153154.yaml delete mode 100644 hatch_build.py delete mode 100644 tests/functional/partial_parsing/fixtures.py delete mode 100644 tests/functional/partial_parsing/test_file_diff.py delete mode 100644 tests/functional/partial_parsing/test_partial_parsing.py delete mode 100644 tests/functional/partial_parsing/test_pp_disabled_config.py delete mode 100644 tests/functional/partial_parsing/test_pp_docs.py delete mode 100644 tests/functional/partial_parsing/test_pp_groups.py delete mode 100644 tests/functional/partial_parsing/test_pp_metrics.py delete mode 100644 tests/functional/partial_parsing/test_pp_vars.py delete mode 100644 tests/functional/partial_parsing/test_versioned_models.py diff --git a/.changes/unreleased/Fixes-20240412-153154.yaml b/.changes/unreleased/Fixes-20240412-153154.yaml new file mode 100644 index 00000000..10bac271 --- /dev/null +++ b/.changes/unreleased/Fixes-20240412-153154.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Determine `psycopg2` based on `platform_system` (Linux or other) +time: 2024-04-12T15:31:54.861201-04:00 +custom: + Author: mikealfare + Issue: "60" diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 691029e3..6bb37ec5 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -39,6 +39,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} cancel-in-progress: true +defaults: + run: + shell: bash + jobs: integration: name: Integration Tests @@ -68,14 +72,12 @@ jobs: - name: Update Adapters and Core branches if: ${{ github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch'}} - shell: bash run: | ./.github/scripts/update_dev_packages.sh \ ${{ inputs.dbt_adapters_branch }} \ ${{ inputs.core_branch }} - name: Setup postgres - shell: bash run: psql -f ./scripts/setup_test_database.sql env: PGHOST: localhost @@ -106,3 +108,27 @@ jobs: source-file: "results.csv" file-name: "integration_results" python-version: ${{ matrix.python-version }} + + psycopg2-check: + name: "Test psycopg2 build version" + runs-on: ${{ matrix.scenario.platform }} + strategy: + fail-fast: false + matrix: + scenario: + - {platform: ubuntu-latest, psycopg2-name: psycopg2} + - {platform: macos-latest, psycopg2-name: psycopg2-binary} + steps: + - name: "Check out repository" + uses: actions/checkout@v4 + + - name: "Test psycopg2 name" + run: | + python -m pip install . + PSYCOPG2_PIP_ENTRY=$(pip list | grep "psycopg2 " || pip list | grep psycopg2-binary) + echo $PSYCOPG2_PIP_ENTRY + PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" + echo $PSYCOPG2_NAME + if [[ "${PSYCOPG2_NAME}" != "${{ matrix.scenario.psycopg2-name }}" ]]; then + exit 1 + fi diff --git a/hatch_build.py b/hatch_build.py deleted file mode 100644 index a44d06c7..00000000 --- a/hatch_build.py +++ /dev/null @@ -1,56 +0,0 @@ -import os -from typing import Any, Dict - -from hatchling.builders.config import BuilderConfig -from hatchling.builders.hooks.plugin.interface import BuildHookInterface -from hatchling.plugin import hookimpl - -BASE_DEPS = [ - # psycopg2 dependency installed in custom hatch_build.py - "dbt-adapters>=0.1.0a1,<2.0", - # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency - "dbt-core>=1.8.0a1", - # installed via dbt-adapters but used directly - "dbt-common>=0.1.0a1,<2.0", - "agate>=1.0,<2.0", -] - -PSYCOPG2_MESSAGE = """ -No package name override was set. -Using 'psycopg2-binary' package to satisfy 'psycopg2' - -If you experience segmentation faults, silent crashes, or installation errors, -consider retrying with the 'DBT_PSYCOPG2_NAME' environment variable set to -'psycopg2'. It may require a compiler toolchain and development libraries! -""".strip() - - -def _dbt_psycopg2_name(): - # if the user chose something, use that - package_name = os.getenv("DBT_PSYCOPG2_NAME", "") - if package_name: - return package_name - - # default to psycopg2-binary for all OSes/versions - print(PSYCOPG2_MESSAGE) - return "psycopg2-binary" - - -class CustomBuildHook(BuildHookInterface[BuilderConfig]): - """ - Custom build hook to install psycopg2 instead of psycopg2-binary based on the presence of `DBT_PSYCOPG2_NAME` env - var. This is necessary as psycopg2-binary is better for local development, but psycopg2 is better for production. - """ - - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - - def initialize(self, version: str, build_data: Dict) -> None: - build_data["dependencies"] = BASE_DEPS - psycopg2_pkg_name = _dbt_psycopg2_name() - build_data["dependencies"].append(f"{psycopg2_pkg_name}>=2.9,<3.0") - - -@hookimpl -def hatch_register_build_hook(): - return CustomBuildHook diff --git a/pyproject.toml b/pyproject.toml index e9d880a2..996ea46a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -dynamic = ["version", "dependencies"] +dynamic = ["version"] name = "dbt-postgres" description = "The set of adapter protocols and base functionality that supports integration with dbt-core" readme = "README.md" @@ -22,6 +22,18 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] +dependencies = [ + # install `psycopg2` on linux (assumed production) + 'psycopg2>=2.9,<3.0; platform_system == "Linux"', + # install `psycopg2-binary` on macos/windows (assumed development) + 'psycopg2-binary>=2.9,<3.0; platform_system != "Linux"', + "dbt-adapters>=0.1.0a1,<2.0", + # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency + "dbt-core>=1.8.0a1", + # installed via dbt-adapters but used directly + "dbt-common>=0.1.0a1,<2.0", + "agate>=1.0,<2.0", +] [project.urls] Homepage = "https://github.com/dbt-labs/dbt-postgres" @@ -43,9 +55,6 @@ packages = ["dbt"] [tool.hatch.version] path = "dbt/adapters/postgres/__version__.py" -[tool.hatch.build.hooks.custom] -path = "./hatch_build.py" - [tool.hatch.envs.default] dependencies = [ "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", diff --git a/tests/functional/partial_parsing/fixtures.py b/tests/functional/partial_parsing/fixtures.py deleted file mode 100644 index f76d90ad..00000000 --- a/tests/functional/partial_parsing/fixtures.py +++ /dev/null @@ -1,1228 +0,0 @@ -local_dependency__dbt_project_yml = """ - -name: 'local_dep' -version: '1.0' -config-version: 2 - -profile: 'default' - -model-paths: ["models"] -analysis-paths: ["analyses"] -test-paths: ["tests"] -seed-paths: ["seeds"] -macro-paths: ["macros"] - -require-dbt-version: '>=0.1.0' - -target-path: "target" # directory which will store compiled SQL files -clean-targets: # directories to be removed by `dbt clean` - - "target" - - "dbt_packages" - - -seeds: - quote_columns: False - -""" - -local_dependency__models__schema_yml = """ -sources: - - name: seed_source - schema: "{{ var('schema_override', target.schema) }}" - tables: - - name: "seed" - columns: - - name: id - data_tests: - - unique - -""" - -local_dependency__models__model_to_import_sql = """ -select * from {{ ref('seed') }} - -""" - -local_dependency__macros__dep_macro_sql = """ -{% macro some_overridden_macro() -%} -100 -{%- endmacro %} - -""" - -local_dependency__seeds__seed_csv = """id -1 -""" - -empty_schema_with_version_yml = """ - -""" - -schema_sources5_yml = """ - -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - -seeds: - - name: rad_customers - description: "Raw customer data" - columns: - - name: id - data_tests: - - unique - - not_null - - name: first_name - - name: last_name - - name: email - - -""" - -my_macro2_sql = """ -{% macro do_something(foo2, bar2) %} - - select - 'foo' as foo2, - 'var' as bar2 - -{% endmacro %} - -""" - -raw_customers_csv = """id,first_name,last_name,email -1,Michael,Perez,mperez0@chronoengine.com -2,Shawn,Mccoy,smccoy1@reddit.com -3,Kathleen,Payne,kpayne2@cargocollective.com -4,Jimmy,Cooper,jcooper3@cargocollective.com -5,Katherine,Rice,krice4@typepad.com -6,Sarah,Ryan,sryan5@gnu.org -7,Martin,Mcdonald,mmcdonald6@opera.com -8,Frank,Robinson,frobinson7@wunderground.com -9,Jennifer,Franklin,jfranklin8@mail.ru -10,Henry,Welch,hwelch9@list-manage.com -""" - -model_three_disabled2_sql = """ -- Disabled model -{{ config(materialized='table', enabled=False) }} - -with source_data as ( - - select 1 as id - union all - select null as id - -) - -select * -from source_data - -""" - -schema_sources4_yml = """ - -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - every_value_is_blue - - name: first_name - - name: last_name - - name: email - -seeds: - - name: raw_customers - description: "Raw customer data" - columns: - - name: id - data_tests: - - unique - - not_null - - name: first_name - - name: last_name - - name: email - - -""" - -env_var_schema_yml = """ - -models: - - name: model_one - config: - materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" - -""" - -my_test_sql = """ -select - * from {{ ref('customers') }} where first_name = '{{ macro_something() }}' - -""" - -empty_schema_yml = """ - -""" - -schema_models_c_yml = """ - -sources: - - name: seed_source - description: "This is a source override" - overrides: local_dep - schema: "{{ var('schema_override', target.schema) }}" - tables: - - name: "seed" - columns: - - name: id - data_tests: - - unique - - not_null - -""" - -env_var_sources_yml = """ -sources: - - name: seed_sources - schema: "{{ target.schema }}" - database: "{{ env_var('ENV_VAR_DATABASE') }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ env_var('ENV_VAR_SEVERITY') }}" - - unique - - name: first_name - - name: last_name - - name: email - - - -""" - -generic_test_edited_sql = """ -{% test is_odd(model, column_name) %} - -with validation as ( - - select - {{ column_name }} as odd_field2 - - from {{ model }} - -), - -validation_errors as ( - - select - odd_field2 - - from validation - -- if this is true, then odd_field is actually even! - where (odd_field2 % 2) = 0 - -) - -select * -from validation_errors - -{% endtest %} -""" - -schema_sources1_yml = """ -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - - - -""" - -schema_sources3_yml = """ - -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - -exposures: - - name: proxy_for_dashboard - description: "This is for the XXX dashboard" - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") - - source("seed_sources", "raw_customers") - - -""" - -my_analysis_sql = """ -select * from customers - -""" - -schema_sources2_yml = """ - -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - -exposures: - - name: proxy_for_dashboard - description: "This is for the XXX dashboard" - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") - - ref("raw_customers") - - source("seed_sources", "raw_customers") - - -""" - -model_color_sql = """ -select 'blue' as fun - -""" - -my_metric_yml = """ -metrics: - - name: new_customers - label: New Customers - model: customers - description: "The number of paid customers who are using the product" - type: simple - type_params: - measure: - name: customers - filter: "{{ Dimension('id__loves_dbt') }} is true" - +meta: - is_okr: True - tags: - - okrs - - - -""" - -env_var_schema2_yml = """ - -models: - - name: model_one - config: - materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" - data_tests: - - check_color: - column_name: fun - color: "env_var('ENV_VAR_COLOR')" - - -""" - -gsm_override_sql = """ -- custom macro -{% macro generate_schema_name(schema_name, node) %} - - {{ schema_name }}_{{ target.schema }} - -{% endmacro %} - -""" - -model_four1_sql = """ -select * from {{ ref('model_three') }} - -""" - -model_one_sql = """ -select 1 as fun - -""" - -metricflow_time_spine_sql = """ -SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day -""" - -env_var_schema3_yml = """ - -models: - - name: model_one - config: - materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" - data_tests: - - check_color: - column_name: fun - color: "env_var('ENV_VAR_COLOR')" - -exposures: - - name: proxy_for_dashboard - description: "This is for the XXX dashboard" - type: "dashboard" - owner: - name: "{{ env_var('ENV_VAR_OWNER') }}" - email: "tester@dashboard.com" - depends_on: - - ref("model_color") - - source("seed_sources", "raw_customers") - -""" - -people_semantic_models_yml = """ -version: 2 - -semantic_models: - - name: semantic_people - model: ref('people') - dimensions: - - name: favorite_color - type: categorical - - name: created_at - type: TIME - type_params: - time_granularity: day - measures: - - name: years_tenure - agg: SUM - expr: tenure - - name: people - agg: count - expr: id - entities: - - name: id - type: primary - defaults: - agg_time_dimension: created_at -""" - -env_var_metrics_yml = """ - -metrics: - - - name: number_of_people - description: Total count of people - label: "Number of people" - type: simple - type_params: - measure: people - meta: - my_meta: '{{ env_var("ENV_VAR_METRICS") }}' - - - name: collective_tenure - description: Total number of years of team experience - label: "Collective tenure" - type: simple - type_params: - measure: - name: years_tenure - filter: "{{ Dimension('id__loves_dbt') }} is true" - -""" - -customers_sql = """ -with source as ( - - select * from {{ source('seed_sources', 'raw_customers') }} - -), - -renamed as ( - - select - id as customer_id, - first_name, - last_name, - email - - from source - -) - -select * from renamed - -""" - -model_four2_sql = """ -select fun from {{ ref('model_one') }} - -""" - -env_var_model_sql = """ -select '{{ env_var('ENV_VAR_TEST') }}' as vartest - -""" - -env_var_model_one_sql = """ -select 'blue' as fun - -""" - -custom_schema_tests2_sql = """ -{% test type_one(model) %} - - select * from ( - - select * from {{ model }} - union all - select * from {{ ref('model_b') }} - - ) as Foo - -{% endtest %} - -{% test type_two(model) %} - - {{ config(severity = "ERROR") }} - - select * from {{ model }} - -{% endtest %} - -""" - -metric_model_a_sql = """ -{% - set metric_list = [ - metric('number_of_people'), - metric('collective_tenure') - ] -%} - -{% if not execute %} - - {% set metric_names = [] %} - {% for m in metric_list %} - {% do metric_names.append(m.metric_name) %} - {% endfor %} - - -- this config does nothing, but it lets us check these values - {{ config(metric_names = metric_names) }} - -{% endif %} - - -select 1 as fun - -""" - -model_b_sql = """ -select 1 as notfun - -""" - -customers2_md = """ -{% docs customer_table %} - -LOTS of customer data - -{% enddocs %} - -""" - -custom_schema_tests1_sql = """ -{% test type_one(model) %} - - select * from ( - - select * from {{ model }} - union all - select * from {{ ref('model_b') }} - - ) as Foo - -{% endtest %} - -{% test type_two(model) %} - - {{ config(severity = "WARN") }} - - select * from {{ model }} - -{% endtest %} - -""" - -people_metrics_yml = """ - -metrics: - - - name: number_of_people - description: Total count of people - label: "Number of people" - type: simple - type_params: - measure: people - meta: - my_meta: 'testing' - - - name: collective_tenure - description: Total number of years of team experience - label: "Collective tenure" - type: simple - type_params: - measure: - name: years_tenure - filter: "{{ Dimension('id__loves_dbt') }} is true" - -""" - -people_sql = """ -select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, true as loves_dbt, 5 as tenure, current_timestamp as created_at -union all -select 1 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at - -""" - -orders_sql = """ -select 1 as id, 101 as user_id, 'pending' as status - -""" - -orders_downstream_sql = """ -select * from {{ ref('orders') }} - -""" - -model_a_sql = """ -select 1 as fun - -""" - -model_three_disabled_sql = """ -{{ config(materialized='table', enabled=False) }} - -with source_data as ( - - select 1 as id - union all - select null as id - -) - -select * -from source_data - -""" - -models_schema2b_yml = """ - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - columns: - - name: id - data_tests: - - not_null - -""" - -env_var_macros_yml = """ -macros: - - name: do_something - description: "This is a test macro" - meta: - some_key: "{{ env_var('ENV_VAR_SOME_KEY') }}" - - -""" - -models_schema4_yml = """ - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - config: - enabled: false - columns: - - name: id - data_tests: - - unique - -""" - -model_two_sql = """ -select 1 as notfun - -""" - -generic_test_schema_yml = """ - -models: - - name: orders - description: "Some order data" - columns: - - name: id - data_tests: - - unique - - is_odd - -""" - -customers1_md = """ -{% docs customer_table %} - -This table contains customer data - -{% enddocs %} - -""" - -model_three_modified_sql = """ -{{ config(materialized='table') }} - -with source_data as ( - - {#- This is model three #} - - select 1 as id - union all - select null as id - -) - -select * -from source_data - -""" - -macros_yml = """ -macros: - - name: do_something - description: "This is a test macro" - -""" - -test_color_sql = """ -{% test check_color(model, column_name, color) %} - - select * - from {{ model }} - where {{ column_name }} = '{{ color }}' - -{% endtest %} - -""" - -models_schema2_yml = """ - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - columns: - - name: id - data_tests: - - unique - -""" - -gsm_override2_sql = """ -- custom macro xxxx -{% macro generate_schema_name(schema_name, node) %} - - {{ schema_name }}_{{ target.schema }} - -{% endmacro %} - -""" - -models_schema3_yml = """ - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - data_tests: - - unique -macros: - - name: do_something - description: "This is a test macro" - -""" - -generic_test_sql = """ -{% test is_odd(model, column_name) %} - -with validation as ( - - select - {{ column_name }} as odd_field - - from {{ model }} - -), - -validation_errors as ( - - select - odd_field - - from validation - -- if this is true, then odd_field is actually even! - where (odd_field % 2) = 0 - -) - -select * -from validation_errors - -{% endtest %} -""" - -env_var_model_test_yml = """ -models: - - name: model_color - columns: - - name: fun - data_tests: - - unique: - enabled: "{{ env_var('ENV_VAR_ENABLED', True) }}" - -""" - -model_three_sql = """ -{{ config(materialized='table') }} - -with source_data as ( - - select 1 as id - union all - select null as id - -) - -select * -from source_data - -""" - -ref_override2_sql = """ -- Macro to override ref xxxx -{% macro ref(modelname) %} -{% do return(builtins.ref(modelname)) %} -{% endmacro %} - -""" - -models_schema1_yml = """ - -models: - - name: model_one - description: "The first model" - -""" - -macros_schema_yml = """ - - -models: - - name: model_a - data_tests: - - type_one - - type_two - -""" - -models_versions_schema_yml = """ - -models: - - name: model_one - description: "The first model" - versions: - - v: 1 - - v: 2 -""" - -models_versions_defined_in_schema_yml = """ - -models: - - name: model_one - description: "The first model" - versions: - - v: 1 - - v: 2 - defined_in: model_one_different -""" - -models_versions_updated_schema_yml = """ - -models: - - name: model_one - latest_version: 1 - description: "The first model" - versions: - - v: 1 - - v: 2 - defined_in: model_one_different -""" - -my_macro_sql = """ -{% macro do_something(foo2, bar2) %} - - select - '{{ foo2 }}' as foo2, - '{{ bar2 }}' as bar2 - -{% endmacro %} - -""" - -snapshot_sql = """ -{% snapshot orders_snapshot %} - -{{ - config( - target_schema=schema, - strategy='check', - unique_key='id', - check_cols=['status'], - ) -}} - -select * from {{ ref('orders') }} - -{% endsnapshot %} - -{% snapshot orders2_snapshot %} - -{{ - config( - target_schema=schema, - strategy='check', - unique_key='id', - check_cols=['order_date'], - ) -}} - -select * from {{ ref('orders') }} - -{% endsnapshot %} - -""" - -models_schema4b_yml = """ - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - config: - enabled: true - columns: - - name: id - data_tests: - - unique - -""" - -test_macro_sql = """ -{% macro macro_something() %} - - {% do return('macro_something') %} - -{% endmacro %} - -""" - -people_metrics2_yml = """ - -metrics: - - - name: number_of_people - description: Total count of people - label: "Number of people" - type: simple - type_params: - measure: people - meta: - my_meta: 'replaced' - - - name: collective_tenure - description: Total number of years of team experience - label: "Collective tenure" - type: simple - type_params: - measure: - name: years_tenure - filter: "{{ Dimension('id__loves_dbt') }} is true" - -""" - -generic_schema_yml = """ - -models: - - name: orders - description: "Some order data" - columns: - - name: id - data_tests: - - unique - -""" - - -groups_schema_yml_one_group = """ - -groups: - - name: test_group - owner: - name: test_group_owner - -models: - - name: orders - description: "Some order data" -""" - - -groups_schema_yml_two_groups = """ - -groups: - - name: test_group - owner: - name: test_group_owner - - name: test_group2 - owner: - name: test_group_owner2 - -models: - - name: orders - description: "Some order data" -""" - - -groups_schema_yml_two_groups_private_orders_valid_access = """ - -groups: - - name: test_group - owner: - name: test_group_owner - - name: test_group2 - owner: - name: test_group_owner2 - -models: - - name: orders - group: test_group - access: private - description: "Some order data" - - name: orders_downstream - group: test_group - description: "Some order data" -""" - -groups_schema_yml_two_groups_private_orders_invalid_access = """ - -groups: - - name: test_group - owner: - name: test_group_owner - - name: test_group2 - owner: - name: test_group_owner2 - -models: - - name: orders - group: test_group2 - access: private - description: "Some order data" - - name: orders_downstream - group: test_group - description: "Some order data" -""" - -groups_schema_yml_one_group_model_in_group2 = """ - -groups: - - name: test_group - owner: - name: test_group_owner - -models: - - name: orders - description: "Some order data" - config: - group: test_group2 -""" - -groups_schema_yml_two_groups_edited = """ - -groups: - - name: test_group - owner: - name: test_group_owner - - name: test_group2_edited - owner: - name: test_group_owner2 - -models: - - name: orders - description: "Some order data" -""" - - -snapshot2_sql = """ -- add a comment -{% snapshot orders_snapshot %} - -{{ - config( - target_schema=schema, - strategy='check', - unique_key='id', - check_cols=['status'], - ) -}} - -select * from {{ ref('orders') }} - -{% endsnapshot %} - -{% snapshot orders2_snapshot %} - -{{ - config( - target_schema=schema, - strategy='check', - unique_key='id', - check_cols=['order_date'], - ) -}} - -select * from {{ ref('orders') }} - -{% endsnapshot %} - -""" - -sources_tests2_sql = """ - -{% test every_value_is_blue(model, column_name) %} - - select * - from {{ model }} - where {{ column_name }} != 99 - -{% endtest %} - - -""" - -people_metrics3_yml = """ - -metrics: - - - name: number_of_people - description: Total count of people - label: "Number of people" - type: simple - type_params: - measure: people - meta: - my_meta: 'replaced' - -""" - -ref_override_sql = """ -- Macro to override ref -{% macro ref(modelname) %} -{% do return(builtins.ref(modelname)) %} -{% endmacro %} - -""" - -test_macro2_sql = """ -{% macro macro_something() %} - - {% do return('some_name') %} - -{% endmacro %} - -""" - -env_var_macro_sql = """ -{% macro do_something(foo2, bar2) %} - - select - '{{ foo2 }}' as foo2, - '{{ bar2 }}' as bar2 - -{% endmacro %} - -""" - -sources_tests1_sql = """ - -{% test every_value_is_blue(model, column_name) %} - - select * - from {{ model }} - where {{ column_name }} = 9999 - -{% endtest %} - - -""" diff --git a/tests/functional/partial_parsing/test_file_diff.py b/tests/functional/partial_parsing/test_file_diff.py deleted file mode 100644 index c7e34780..00000000 --- a/tests/functional/partial_parsing/test_file_diff.py +++ /dev/null @@ -1,63 +0,0 @@ -import os - -from dbt.tests.util import run_dbt, write_artifact, write_file -import pytest - -from tests.functional.partial_parsing.fixtures import model_one_sql, model_two_sql - - -first_file_diff = { - "deleted": [], - "changed": [], - "added": [{"path": "models/model_one.sql", "content": "select 1 as fun"}], -} - - -second_file_diff = { - "deleted": [], - "changed": [], - "added": [{"path": "models/model_two.sql", "content": "select 123 as notfun"}], -} - - -class TestFileDiffPaths: - def test_file_diffs(self, project): - os.environ["DBT_PP_FILE_DIFF_TEST"] = "true" - - run_dbt(["deps"]) - run_dbt(["seed"]) - - # We start with an empty project - results = run_dbt() - - write_artifact(first_file_diff, "file_diff.json") - results = run_dbt() - assert len(results) == 1 - - write_artifact(second_file_diff, "file_diff.json") - results = run_dbt() - assert len(results) == 2 - - -class TestFileDiffs: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - def test_no_file_diffs(self, project): - # We start with a project with one model - manifest = run_dbt(["parse"]) - assert len(manifest.nodes) == 1 - - # add a model file - write_file(model_two_sql, project.project_root, "models", "model_two.sql") - - # parse without computing a file diff - manifest = run_dbt(["--partial-parse", "--no-partial-parse-file-diff", "parse"]) - assert len(manifest.nodes) == 1 - - # default behaviour - parse with computing a file diff - manifest = run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 2 diff --git a/tests/functional/partial_parsing/test_partial_parsing.py b/tests/functional/partial_parsing/test_partial_parsing.py deleted file mode 100644 index eb09dd32..00000000 --- a/tests/functional/partial_parsing/test_partial_parsing.py +++ /dev/null @@ -1,824 +0,0 @@ -import os -import re -from unittest import mock - -from dbt.tests.fixtures.project import write_project_files -from dbt.tests.util import ( - get_manifest, - rename_dir, - rm_file, - write_file, -) -from dbt.contracts.files import ParseFileType -from dbt.contracts.results import TestStatus -from dbt.plugins.manifest import ModelNodeArgs, PluginNodes -from dbt_common.exceptions import CompilationError -import pytest - -from tests.functional.partial_parsing.fixtures import ( - custom_schema_tests1_sql, - custom_schema_tests2_sql, - customers_sql, - customers1_md, - customers2_md, - empty_schema_with_version_yml, - empty_schema_yml, - generic_schema_yml, - generic_test_edited_sql, - generic_test_schema_yml, - generic_test_sql, - gsm_override_sql, - gsm_override2_sql, - local_dependency__dbt_project_yml, - local_dependency__macros__dep_macro_sql, - local_dependency__models__model_to_import_sql, - local_dependency__models__schema_yml, - local_dependency__seeds__seed_csv, - macros_schema_yml, - macros_yml, - model_a_sql, - model_b_sql, - model_four1_sql, - model_four2_sql, - model_one_sql, - model_three_disabled_sql, - model_three_disabled2_sql, - model_three_modified_sql, - model_three_sql, - model_two_sql, - models_schema1_yml, - models_schema2_yml, - models_schema2b_yml, - models_schema3_yml, - models_schema4_yml, - models_schema4b_yml, - my_analysis_sql, - my_macro_sql, - my_macro2_sql, - my_test_sql, - orders_sql, - raw_customers_csv, - ref_override_sql, - ref_override2_sql, - schema_models_c_yml, - schema_sources1_yml, - schema_sources2_yml, - schema_sources3_yml, - schema_sources4_yml, - schema_sources5_yml, - snapshot_sql, - snapshot2_sql, - sources_tests1_sql, - sources_tests2_sql, - test_macro_sql, - test_macro2_sql, -) -from tests.functional.utils import ( - run_dbt, - run_dbt_and_capture, - up_one, -) - - -os.environ["DBT_PP_TEST"] = "true" - - -def normalize(path): - return os.path.normcase(os.path.normpath(path)) - - -class TestModels: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - def test_pp_models(self, project): - # initial run - # run_dbt(['clean']) - results = run_dbt(["run"]) - assert len(results) == 1 - - # add a model file - write_file(model_two_sql, project.project_root, "models", "model_two.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - # add a schema file - write_file(models_schema1_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert "model.test.model_one" in manifest.nodes - model_one_node = manifest.nodes["model.test.model_one"] - assert model_one_node.description == "The first model" - assert model_one_node.patch_path == "test://" + normalize("models/schema.yml") - - # add a model and a schema file (with a test) at the same time - write_file(models_schema2_yml, project.project_root, "models", "schema.yml") - write_file(model_three_sql, project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "test"], expect_pass=False) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - project_files = [f for f in manifest.files if f.startswith("test://")] - assert len(project_files) == 4 - model_3_file_id = "test://" + normalize("models/model_three.sql") - assert model_3_file_id in manifest.files - model_three_file = manifest.files[model_3_file_id] - assert model_three_file.parse_file_type == ParseFileType.Model - assert type(model_three_file).__name__ == "SourceFile" - model_three_node = manifest.nodes[model_three_file.nodes[0]] - schema_file_id = "test://" + normalize("models/schema.yml") - assert model_three_node.patch_path == schema_file_id - assert model_three_node.description == "The third model" - schema_file = manifest.files[schema_file_id] - assert type(schema_file).__name__ == "SchemaSourceFile" - assert len(schema_file.data_tests) == 1 - tests = schema_file.get_all_test_ids() - assert tests == ["test.test.unique_model_three_id.6776ac8160"] - unique_test_id = tests[0] - assert unique_test_id in manifest.nodes - - # modify model sql file, ensure description still there - write_file(model_three_modified_sql, project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - model_id = "model.test.model_three" - assert model_id in manifest.nodes - model_three_node = manifest.nodes[model_id] - assert model_three_node.description == "The third model" - - # Change the model 3 test from unique to not_null - write_file(models_schema2b_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "test"], expect_pass=False) - manifest = get_manifest(project.project_root) - schema_file_id = "test://" + normalize("models/schema.yml") - schema_file = manifest.files[schema_file_id] - tests = schema_file.get_all_test_ids() - assert tests == ["test.test.not_null_model_three_id.3162ce0a6f"] - not_null_test_id = tests[0] - assert not_null_test_id in manifest.nodes.keys() - assert unique_test_id not in manifest.nodes.keys() - assert len(results) == 1 - - # go back to previous version of schema file, removing patch, test, and model for model three - write_file(models_schema1_yml, project.project_root, "models", "schema.yml") - rm_file(project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - # remove schema file, still have 3 models - write_file(model_three_sql, project.project_root, "models", "model_three.sql") - rm_file(project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - manifest = get_manifest(project.project_root) - schema_file_id = "test://" + normalize("models/schema.yml") - assert schema_file_id not in manifest.files - project_files = [f for f in manifest.files if f.startswith("test://")] - assert len(project_files) == 3 - - # Put schema file back and remove a model - # referred to in schema file - write_file(models_schema2_yml, project.project_root, "models", "schema.yml") - rm_file(project.project_root, "models", "model_three.sql") - with pytest.raises(CompilationError): - results = run_dbt(["--partial-parse", "--warn-error", "run"]) - - # Put model back again - write_file(model_three_sql, project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Add model four refing model three - write_file(model_four1_sql, project.project_root, "models", "model_four.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 4 - - # Remove model_three and change model_four to ref model_one - # and change schema file to remove model_three - rm_file(project.project_root, "models", "model_three.sql") - write_file(model_four2_sql, project.project_root, "models", "model_four.sql") - write_file(models_schema1_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Remove model four, put back model three, put back schema file - write_file(model_three_sql, project.project_root, "models", "model_three.sql") - write_file(models_schema2_yml, project.project_root, "models", "schema.yml") - rm_file(project.project_root, "models", "model_four.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # disable model three in the schema file - write_file(models_schema4_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - # update enabled config to be true for model three in the schema file - write_file(models_schema4b_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # disable model three in the schema file again - write_file(models_schema4_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - # remove disabled config for model three in the schema file to check it gets enabled - write_file(models_schema4b_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Add a macro - write_file(my_macro_sql, project.project_root, "macros", "my_macro.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - manifest = get_manifest(project.project_root) - macro_id = "macro.test.do_something" - assert macro_id in manifest.macros - - # Modify the macro - write_file(my_macro2_sql, project.project_root, "macros", "my_macro.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Add a macro patch - write_file(models_schema3_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Remove the macro - rm_file(project.project_root, "macros", "my_macro.sql") - with pytest.raises(CompilationError): - results = run_dbt(["--partial-parse", "--warn-error", "run"]) - - # put back macro file, got back to schema file with no macro - # add separate macro patch schema file - write_file(models_schema2_yml, project.project_root, "models", "schema.yml") - write_file(my_macro_sql, project.project_root, "macros", "my_macro.sql") - write_file(macros_yml, project.project_root, "macros", "macros.yml") - results = run_dbt(["--partial-parse", "run"]) - - # delete macro and schema file - rm_file(project.project_root, "macros", "my_macro.sql") - rm_file(project.project_root, "macros", "macros.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Add an empty schema file - write_file(empty_schema_yml, project.project_root, "models", "eschema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Add version to empty schema file - write_file(empty_schema_with_version_yml, project.project_root, "models", "eschema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Disable model_three - write_file(model_three_disabled_sql, project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - model_id = "model.test.model_three" - assert model_id in manifest.disabled - assert model_id not in manifest.nodes - - # Edit disabled model three - write_file(model_three_disabled2_sql, project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - model_id = "model.test.model_three" - assert model_id in manifest.disabled - assert model_id not in manifest.nodes - - # Remove disabled from model three - write_file(model_three_sql, project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - manifest = get_manifest(project.project_root) - model_id = "model.test.model_three" - assert model_id in manifest.nodes - assert model_id not in manifest.disabled - - -class TestSources: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - def test_pp_sources(self, project): - # initial run - write_file(raw_customers_csv, project.project_root, "seeds", "raw_customers.csv") - write_file(sources_tests1_sql, project.project_root, "macros", "tests.sql") - results = run_dbt(["run"]) - assert len(results) == 1 - - # Partial parse running 'seed' - run_dbt(["--partial-parse", "seed"]) - manifest = get_manifest(project.project_root) - seed_file_id = "test://" + normalize("seeds/raw_customers.csv") - assert seed_file_id in manifest.files - - # Add another seed file - write_file(raw_customers_csv, project.project_root, "seeds", "more_customers.csv") - run_dbt(["--partial-parse", "run"]) - seed_file_id = "test://" + normalize("seeds/more_customers.csv") - manifest = get_manifest(project.project_root) - assert seed_file_id in manifest.files - seed_id = "seed.test.more_customers" - assert seed_id in manifest.nodes - - # Remove seed file and add a schema files with a source referring to raw_customers - rm_file(project.project_root, "seeds", "more_customers.csv") - write_file(schema_sources1_yml, project.project_root, "models", "sources.yml") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - assert len(manifest.sources) == 1 - file_id = "test://" + normalize("models/sources.yml") - assert file_id in manifest.files - - # add a model referring to raw_customers source - write_file(customers_sql, project.project_root, "models", "customers.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - # remove sources schema file - rm_file(project.project_root, "models", "sources.yml") - with pytest.raises(CompilationError): - results = run_dbt(["--partial-parse", "run"]) - - # put back sources and add an exposures file - write_file(schema_sources2_yml, project.project_root, "models", "sources.yml") - results = run_dbt(["--partial-parse", "run"]) - - # remove seed referenced in exposures file - rm_file(project.project_root, "seeds", "raw_customers.csv") - with pytest.raises(CompilationError): - results = run_dbt(["--partial-parse", "run"]) - - # put back seed and remove depends_on from exposure - write_file(raw_customers_csv, project.project_root, "seeds", "raw_customers.csv") - write_file(schema_sources3_yml, project.project_root, "models", "sources.yml") - results = run_dbt(["--partial-parse", "run"]) - - # Add seed config with test to schema.yml, remove exposure - write_file(schema_sources4_yml, project.project_root, "models", "sources.yml") - results = run_dbt(["--partial-parse", "run"]) - - # Change seed name to wrong name - write_file(schema_sources5_yml, project.project_root, "models", "sources.yml") - with pytest.raises(CompilationError): - results = run_dbt(["--partial-parse", "--warn-error", "run"]) - - # Put back seed name to right name - write_file(schema_sources4_yml, project.project_root, "models", "sources.yml") - results = run_dbt(["--partial-parse", "run"]) - - # Add docs file customers.md - write_file(customers1_md, project.project_root, "models", "customers.md") - results = run_dbt(["--partial-parse", "run"]) - - # Change docs file customers.md - write_file(customers2_md, project.project_root, "models", "customers.md") - results = run_dbt(["--partial-parse", "run"]) - - # Delete docs file - rm_file(project.project_root, "models", "customers.md") - results = run_dbt(["--partial-parse", "run"]) - - # Add a data test - write_file(test_macro_sql, project.project_root, "macros", "test-macro.sql") - write_file(my_test_sql, project.project_root, "tests", "my_test.sql") - results = run_dbt(["--partial-parse", "test"]) - manifest = get_manifest(project.project_root) - assert len(manifest.nodes) == 9 - test_id = "test.test.my_test" - assert test_id in manifest.nodes - - # Change macro that data test depends on - write_file(test_macro2_sql, project.project_root, "macros", "test-macro.sql") - results = run_dbt(["--partial-parse", "test"]) - manifest = get_manifest(project.project_root) - - # Add an analysis - write_file(my_analysis_sql, project.project_root, "analyses", "my_analysis.sql") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - - # Remove data test - rm_file(project.project_root, "tests", "my_test.sql") - results = run_dbt(["--partial-parse", "test"]) - manifest = get_manifest(project.project_root) - assert len(manifest.nodes) == 9 - - # Remove analysis - rm_file(project.project_root, "analyses", "my_analysis.sql") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - assert len(manifest.nodes) == 8 - - # Change source test - write_file(sources_tests2_sql, project.project_root, "macros", "tests.sql") - results = run_dbt(["--partial-parse", "run"]) - - -class TestPartialParsingDependency: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root): - local_dependency_files = { - "dbt_project.yml": local_dependency__dbt_project_yml, - "models": { - "schema.yml": local_dependency__models__schema_yml, - "model_to_import.sql": local_dependency__models__model_to_import_sql, - }, - "macros": {"dep_macro.sql": local_dependency__macros__dep_macro_sql}, - "seeds": {"seed.csv": local_dependency__seeds__seed_csv}, - } - write_project_files(project_root, "local_dependency", local_dependency_files) - - @pytest.fixture(scope="class") - def packages(self): - return {"packages": [{"local": "local_dependency"}]} - - def test_parsing_with_dependency(self, project): - run_dbt(["clean"]) - run_dbt(["deps"]) - run_dbt(["seed"]) - run_dbt(["run"]) - - # Add a source override - write_file(schema_models_c_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert len(manifest.sources) == 1 - source_id = "source.local_dep.seed_source.seed" - assert source_id in manifest.sources - # We have 1 root model, 1 local_dep model, 1 local_dep seed, 1 local_dep source test, 2 root source tests - assert len(manifest.nodes) == 5 - test_id = "test.local_dep.source_unique_seed_source_seed_id.afa94935ed" - assert test_id in manifest.nodes - - # Remove a source override - rm_file(project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - assert len(manifest.sources) == 1 - - -class TestNestedMacros: - @pytest.fixture(scope="class") - def models(self): - return { - "model_a.sql": model_a_sql, - "model_b.sql": model_b_sql, - "schema.yml": macros_schema_yml, - } - - @pytest.fixture(scope="class") - def macros(self): - return { - "custom_schema_tests.sql": custom_schema_tests1_sql, - } - - def test_nested_macros(self, project): - results = run_dbt() - assert len(results) == 2 - manifest = get_manifest(project.project_root) - macro_child_map = manifest.build_macro_child_map() - macro_unique_id = "macro.test.test_type_two" - assert macro_unique_id in macro_child_map - - results = run_dbt(["test"], expect_pass=False) - results = sorted(results, key=lambda r: r.node.name) - assert len(results) == 2 - # type_one_model_a_ - assert results[0].status == TestStatus.Fail - assert re.search(r"union all", results[0].node.compiled_code) - # type_two_model_a_ - assert results[1].status == TestStatus.Warn - assert results[1].node.config.severity == "WARN" - - write_file( - custom_schema_tests2_sql, project.project_root, "macros", "custom_schema_tests.sql" - ) - results = run_dbt(["--partial-parse", "test"], expect_pass=False) - manifest = get_manifest(project.project_root) - test_node_id = "test.test.type_two_model_a_.842bc6c2a7" - assert test_node_id in manifest.nodes - results = sorted(results, key=lambda r: r.node.name) - assert len(results) == 2 - # type_two_model_a_ - assert results[1].status == TestStatus.Fail - assert results[1].node.config.severity == "ERROR" - - -class TestSkipMacros: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - "eschema.yml": empty_schema_yml, - } - - def test_skip_macros(self, project): - # initial run so we have a msgpack file - # includes empty_schema file for bug #4850 - results = run_dbt() - - # add a new ref override macro - write_file(ref_override_sql, project.project_root, "macros", "ref_override.sql") - results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) - assert "Starting full parse." in log_output - - # modify a ref override macro - write_file(ref_override2_sql, project.project_root, "macros", "ref_override.sql") - results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) - assert "Starting full parse." in log_output - - # remove a ref override macro - rm_file(project.project_root, "macros", "ref_override.sql") - results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) - assert "Starting full parse." in log_output - - # custom generate_schema_name macro - write_file(gsm_override_sql, project.project_root, "macros", "gsm_override.sql") - results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) - assert "Starting full parse." in log_output - - # change generate_schema_name macro - write_file(gsm_override2_sql, project.project_root, "macros", "gsm_override.sql") - results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) - assert "Starting full parse." in log_output - - -class TestSnapshots: - @pytest.fixture(scope="class") - def models(self): - return { - "orders.sql": orders_sql, - } - - def test_pp_snapshots(self, project): - # initial run - results = run_dbt() - assert len(results) == 1 - - # add snapshot - write_file(snapshot_sql, project.project_root, "snapshots", "snapshot.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - snapshot_id = "snapshot.test.orders_snapshot" - assert snapshot_id in manifest.nodes - snapshot2_id = "snapshot.test.orders2_snapshot" - assert snapshot2_id in manifest.nodes - - # run snapshot - results = run_dbt(["--partial-parse", "snapshot"]) - assert len(results) == 2 - - # modify snapshot - write_file(snapshot2_sql, project.project_root, "snapshots", "snapshot.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 - - # delete snapshot - rm_file(project.project_root, "snapshots", "snapshot.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 - - -class TestTests: - @pytest.fixture(scope="class") - def models(self): - return { - "orders.sql": orders_sql, - "schema.yml": generic_schema_yml, - } - - @pytest.fixture(scope="class") - def tests(self): - # Make sure "generic" directory is created - return {"generic": {"readme.md": ""}} - - def test_pp_generic_tests(self, project): - # initial run - results = run_dbt() - assert len(results) == 1 - manifest = get_manifest(project.project_root) - expected_nodes = ["model.test.orders", "test.test.unique_orders_id.1360ecc70e"] - assert expected_nodes == list(manifest.nodes.keys()) - - # add generic test in test-path - write_file(generic_test_sql, project.project_root, "tests", "generic", "generic_test.sql") - write_file(generic_test_schema_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - test_id = "test.test.is_odd_orders_id.82834fdc5b" - assert test_id in manifest.nodes - expected_nodes = [ - "model.test.orders", - "test.test.unique_orders_id.1360ecc70e", - "test.test.is_odd_orders_id.82834fdc5b", - ] - assert expected_nodes == list(manifest.nodes.keys()) - - # edit generic test in test-path - write_file( - generic_test_edited_sql, project.project_root, "tests", "generic", "generic_test.sql" - ) - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - test_id = "test.test.is_odd_orders_id.82834fdc5b" - assert test_id in manifest.nodes - expected_nodes = [ - "model.test.orders", - "test.test.unique_orders_id.1360ecc70e", - "test.test.is_odd_orders_id.82834fdc5b", - ] - assert expected_nodes == list(manifest.nodes.keys()) - - -class TestExternalModels: - @pytest.fixture(scope="class") - def external_model_node(self): - return ModelNodeArgs( - name="external_model", - package_name="external", - identifier="test_identifier", - schema="test_schema", - ) - - @pytest.fixture(scope="class") - def external_model_node_versioned(self): - return ModelNodeArgs( - name="external_model_versioned", - package_name="external", - identifier="test_identifier_v1", - schema="test_schema", - version=1, - ) - - @pytest.fixture(scope="class") - def external_model_node_depends_on(self): - return ModelNodeArgs( - name="external_model_depends_on", - package_name="external", - identifier="test_identifier_depends_on", - schema="test_schema", - depends_on_nodes=["model.external.external_model_depends_on_parent"], - ) - - @pytest.fixture(scope="class") - def external_model_node_depends_on_parent(self): - return ModelNodeArgs( - name="external_model_depends_on_parent", - package_name="external", - identifier="test_identifier_depends_on_parent", - schema="test_schema", - ) - - @pytest.fixture(scope="class") - def models(self): - return {"model_one.sql": model_one_sql} - - @mock.patch("dbt.plugins.get_plugin_manager") - def test_pp_external_models( - self, - get_plugin_manager, - project, - external_model_node, - external_model_node_versioned, - external_model_node_depends_on, - external_model_node_depends_on_parent, - ): - # initial plugin - one external model - external_nodes = PluginNodes() - external_nodes.add_model(external_model_node) - get_plugin_manager.return_value.get_nodes.return_value = external_nodes - - # initial parse - manifest = run_dbt(["parse"]) - assert len(manifest.nodes) == 2 - assert set(manifest.nodes.keys()) == { - "model.external.external_model", - "model.test.model_one", - } - assert len(manifest.external_node_unique_ids) == 1 - assert manifest.external_node_unique_ids == ["model.external.external_model"] - - # add a model file - write_file(model_two_sql, project.project_root, "models", "model_two.sql") - manifest = run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 3 - - # add an external model - external_nodes.add_model(external_model_node_versioned) - manifest = run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 4 - assert len(manifest.external_node_unique_ids) == 2 - - # add a model file that depends on external model - write_file( - "SELECT * FROM {{ref('external', 'external_model')}}", - project.project_root, - "models", - "model_depends_on_external.sql", - ) - manifest = run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 5 - assert len(manifest.external_node_unique_ids) == 2 - - # remove a model file that depends on external model - rm_file(project.project_root, "models", "model_depends_on_external.sql") - manifest = run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 4 - - # add an external node with depends on - external_nodes.add_model(external_model_node_depends_on) - external_nodes.add_model(external_model_node_depends_on_parent) - manifest = run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 6 - assert len(manifest.external_node_unique_ids) == 4 - - # skip files parsing - ensure no issues - run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 6 - assert len(manifest.external_node_unique_ids) == 4 - - -class TestPortablePartialParsing: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - @pytest.fixture(scope="class") - def packages(self): - return {"packages": [{"local": "local_dependency"}]} - - @pytest.fixture(scope="class") - def local_dependency_files(self): - return { - "dbt_project.yml": local_dependency__dbt_project_yml, - "models": { - "schema.yml": local_dependency__models__schema_yml, - "model_to_import.sql": local_dependency__models__model_to_import_sql, - }, - "macros": {"dep_macro.sql": local_dependency__macros__dep_macro_sql}, - "seeds": {"seed.csv": local_dependency__seeds__seed_csv}, - } - - def rename_project_root(self, project, new_project_root): - with up_one(new_project_root): - rename_dir(project.project_root, new_project_root) - project.project_root = new_project_root - # flags.project_dir is set during the project test fixture, and is persisted across run_dbt calls, - # so it needs to be reset between invocations - # flags.set_from_args(Namespace(PROJECT_DIR=new_project_root), None) - - @pytest.fixture(scope="class", autouse=True) - def initial_run_and_rename_project_dir(self, project, local_dependency_files): - initial_project_root = project.project_root - renamed_project_root = os.path.join(project.project_root.dirname, "renamed_project_dir") - - write_project_files(project.project_root, "local_dependency", local_dependency_files) - - # initial run - run_dbt(["deps"]) - assert len(run_dbt(["seed"])) == 1 - assert len(run_dbt(["run"])) == 2 - - self.rename_project_root(project, renamed_project_root) - yield - self.rename_project_root(project, initial_project_root) - - def test_pp_renamed_project_dir_unchanged_project_contents(self, project): - # partial parse same project in new absolute dir location, using partial_parse.msgpack created in previous dir - run_dbt(["deps"]) - assert len(run_dbt(["--partial-parse", "seed"])) == 1 - assert len(run_dbt(["--partial-parse", "run"])) == 2 - - def test_pp_renamed_project_dir_changed_project_contents(self, project): - write_file(model_two_sql, project.project_root, "models", "model_two.sql") - - # partial parse changed project in new absolute dir location, using partial_parse.msgpack created in previous dir - run_dbt(["deps"]) - len(run_dbt(["--partial-parse", "seed"])) == 1 - len(run_dbt(["--partial-parse", "run"])) == 3 diff --git a/tests/functional/partial_parsing/test_pp_disabled_config.py b/tests/functional/partial_parsing/test_pp_disabled_config.py deleted file mode 100644 index 8a4ece9d..00000000 --- a/tests/functional/partial_parsing/test_pp_disabled_config.py +++ /dev/null @@ -1,224 +0,0 @@ -from dbt.tests.util import get_manifest, run_dbt, write_file -import pytest - - -model_one_sql = """ -select 1 as fun -""" - -metricflow_time_spine_sql = """ -SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day -""" - -schema1_yml = """ -version: 2 - -models: - - name: model_one - -semantic_models: - - name: semantic_people - model: ref('model_one') - dimensions: - - name: created_at - type: TIME - type_params: - time_granularity: day - measures: - - name: people - agg: count - expr: fun - entities: - - name: fun - type: primary - defaults: - agg_time_dimension: created_at - -metrics: - - - name: number_of_people - label: "Number of people" - description: Total count of people - type: simple - type_params: - measure: people - meta: - my_meta: 'testing' - -exposures: - - name: proxy_for_dashboard - description: "My Exposure" - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") -""" - -schema2_yml = """ -version: 2 - -models: - - name: model_one - -semantic_models: - - name: semantic_people - model: ref('model_one') - dimensions: - - name: created_at - type: TIME - type_params: - time_granularity: day - measures: - - name: people - agg: count - expr: fun - entities: - - name: fun - type: primary - defaults: - agg_time_dimension: created_at - -metrics: - - - name: number_of_people - label: "Number of people" - description: Total count of people - config: - enabled: false - type: simple - type_params: - measure: people - meta: - my_meta: 'testing' - -exposures: - - name: proxy_for_dashboard - description: "My Exposure" - config: - enabled: false - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") -""" - -schema3_yml = """ -version: 2 - -models: - - name: model_one - -semantic_models: - - name: semantic_people - model: ref('model_one') - dimensions: - - name: created_at - type: TIME - type_params: - time_granularity: day - measures: - - name: people - agg: count - expr: fun - entities: - - name: fun - type: primary - defaults: - agg_time_dimension: created_at - -metrics: - - - name: number_of_people - label: "Number of people" - description: Total count of people - type: simple - type_params: - measure: people - meta: - my_meta: 'testing' -""" - -schema4_yml = """ -version: 2 - -models: - - name: model_one - -exposures: - - name: proxy_for_dashboard - description: "My Exposure" - config: - enabled: false - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") -""" - - -class TestDisabled: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - "metricflow_time_spine.sql": metricflow_time_spine_sql, - "schema.yml": schema1_yml, - } - - def test_pp_disabled(self, project): - expected_exposure = "exposure.test.proxy_for_dashboard" - expected_metric = "metric.test.number_of_people" - - run_dbt(["seed"]) - manifest = run_dbt(["parse"]) - - assert expected_exposure in manifest.exposures - assert expected_metric in manifest.metrics - assert expected_exposure not in manifest.disabled - assert expected_metric not in manifest.disabled - - # Update schema file with disabled metric and exposure - write_file(schema2_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert expected_exposure not in manifest.exposures - assert expected_metric not in manifest.metrics - assert expected_exposure in manifest.disabled - assert expected_metric in manifest.disabled - - # Update schema file with enabled metric and exposure - write_file(schema1_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert expected_exposure in manifest.exposures - assert expected_metric in manifest.metrics - assert expected_exposure not in manifest.disabled - assert expected_metric not in manifest.disabled - - # Update schema file - remove exposure, enable metric - write_file(schema3_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert expected_exposure not in manifest.exposures - assert expected_metric in manifest.metrics - assert expected_exposure not in manifest.disabled - assert expected_metric not in manifest.disabled - - # Update schema file - add back exposure, remove metric - write_file(schema4_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert expected_exposure not in manifest.exposures - assert expected_metric not in manifest.metrics - assert expected_exposure in manifest.disabled - assert expected_metric not in manifest.disabled diff --git a/tests/functional/partial_parsing/test_pp_docs.py b/tests/functional/partial_parsing/test_pp_docs.py deleted file mode 100644 index 5df08d4d..00000000 --- a/tests/functional/partial_parsing/test_pp_docs.py +++ /dev/null @@ -1,257 +0,0 @@ -from dbt.tests.util import get_manifest, rm_file, run_dbt, write_file -import pytest - - -model_one_sql = """ -select 1 as fun -""" - -raw_customers_csv = """id,first_name,last_name,email -1,Michael,Perez,mperez0@chronoengine.com -2,Shawn,Mccoy,smccoy1@reddit.com -3,Kathleen,Payne,kpayne2@cargocollective.com -4,Jimmy,Cooper,jcooper3@cargocollective.com -5,Katherine,Rice,krice4@typepad.com -6,Sarah,Ryan,sryan5@gnu.org -7,Martin,Mcdonald,mmcdonald6@opera.com -8,Frank,Robinson,frobinson7@wunderground.com -9,Jennifer,Franklin,jfranklin8@mail.ru -10,Henry,Welch,hwelch9@list-manage.com -""" - -my_macro_sql = """ -{% macro my_macro(something) %} - - select - '{{ something }}' as something2 - -{% endmacro %} - -""" - -customers1_md = """ -{% docs customer_table %} - -This table contains customer data - -{% enddocs %} -""" - -customers2_md = """ -{% docs customer_table %} - -LOTS of customer data - -{% enddocs %} - -""" - -schema1_yml = """ -version: 2 - -models: - - name: model_one - description: "{{ doc('customer_table') }}" -""" - -schema2_yml = """ -version: 2 - -models: - - name: model_one - description: "{{ doc('customer_table') }}" - -macros: - - name: my_macro - description: "{{ doc('customer_table') }}" - -sources: - - name: seed_sources - description: "{{ doc('customer_table') }}" - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - -exposures: - - name: proxy_for_dashboard - description: "{{ doc('customer_table') }}" - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") - - ref("raw_customers") - - source("seed_sources", "raw_customers") -""" - - -class TestDocs: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - @pytest.fixture(scope="class") - def seeds(self): - return { - "raw_customers.csv": raw_customers_csv, - } - - @pytest.fixture(scope="class") - def macros(self): - return { - "my_macro.sql": my_macro_sql, - } - - def test_pp_docs(self, project): - run_dbt(["seed"]) - results = run_dbt(["run"]) - assert len(results) == 1 - - # Add docs file customers.md - write_file(customers1_md, project.project_root, "models", "customers.md") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - assert len(manifest.docs) == 2 - - # Add schema file with 'docs' description - write_file(schema1_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - assert len(manifest.docs) == 2 - doc_id = "doc.test.customer_table" - assert doc_id in manifest.docs - doc = manifest.docs[doc_id] - doc_file_id = doc.file_id - assert doc_file_id in manifest.files - source_file = manifest.files[doc_file_id] - assert len(source_file.nodes) == 1 - model_one_id = "model.test.model_one" - assert model_one_id in source_file.nodes - model_node = manifest.nodes[model_one_id] - assert model_node.description == "This table contains customer data" - - # Update the doc file - write_file(customers2_md, project.project_root, "models", "customers.md") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - assert len(manifest.docs) == 2 - assert model_one_id in manifest.nodes - model_node = manifest.nodes[model_one_id] - assert "LOTS" in model_node.description - - # Add a macro patch, source and exposure with doc - write_file(schema2_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - doc_file = manifest.files[doc_file_id] - expected_nodes = [ - "model.test.model_one", - "source.test.seed_sources.raw_customers", - "macro.test.my_macro", - "exposure.test.proxy_for_dashboard", - ] - assert expected_nodes == doc_file.nodes - source_id = "source.test.seed_sources.raw_customers" - assert manifest.sources[source_id].source_description == "LOTS of customer data" - macro_id = "macro.test.my_macro" - assert manifest.macros[macro_id].description == "LOTS of customer data" - exposure_id = "exposure.test.proxy_for_dashboard" - assert manifest.exposures[exposure_id].description == "LOTS of customer data" - - # update the doc file again - write_file(customers1_md, project.project_root, "models", "customers.md") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - source_file = manifest.files[doc_file_id] - assert model_one_id in source_file.nodes - model_node = manifest.nodes[model_one_id] - assert model_node.description == "This table contains customer data" - assert ( - manifest.sources[source_id].source_description == "This table contains customer data" - ) - assert manifest.macros[macro_id].description == "This table contains customer data" - assert manifest.exposures[exposure_id].description == "This table contains customer data" - - # check that _lock is working - with manifest._lock: - assert manifest._lock - - -my_model_yml = """ -version: 2 -models: - - name: my_model - columns: - - name: id - description: "{{ doc('whatever') }}" -""" - -my_model_no_description_yml = """ -version: 2 -models: - - name: my_model - columns: - - name: id -""" - -my_model_md = """ -{% docs whatever %} - cool stuff -{% enddocs %} -""" - - -class TestDocsRemoveReplace: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": "select 1 as id", - "my_model.yml": my_model_yml, - "my_model.md": my_model_md, - } - - def test_remove_replace(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - doc_id = "doc.test.whatever" - assert doc_id in manifest.docs - doc = manifest.docs[doc_id] - doc_file = manifest.files[doc.file_id] - - model_id = "model.test.my_model" - assert model_id in manifest.nodes - - assert doc_file.nodes == [model_id] - - model = manifest.nodes[model_id] - model_file_id = model.file_id - assert model_file_id in manifest.files - - # remove the doc file - rm_file(project.project_root, "models", "my_model.md") - # remove description from schema file - write_file(my_model_no_description_yml, project.project_root, "models", "my_model.yml") - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert doc_id not in manifest.docs - # The bug was that the file still existed in manifest.files - assert doc.file_id not in manifest.files - - # put back the doc file - write_file(my_model_md, project.project_root, "models", "my_model.md") - # put back the description in the schema file - write_file(my_model_yml, project.project_root, "models", "my_model.yml") - run_dbt(["parse"]) diff --git a/tests/functional/partial_parsing/test_pp_groups.py b/tests/functional/partial_parsing/test_pp_groups.py deleted file mode 100644 index f7577683..00000000 --- a/tests/functional/partial_parsing/test_pp_groups.py +++ /dev/null @@ -1,155 +0,0 @@ -from dbt.exceptions import ParsingError -from dbt.tests.util import get_manifest, run_dbt, write_file -import pytest - -from tests.functional.partial_parsing.fixtures import ( - groups_schema_yml_one_group, - groups_schema_yml_one_group_model_in_group2, - groups_schema_yml_two_groups, - groups_schema_yml_two_groups_edited, - groups_schema_yml_two_groups_private_orders_invalid_access, - groups_schema_yml_two_groups_private_orders_valid_access, - orders_downstream_sql, - orders_sql, -) - - -class TestGroups: - @pytest.fixture(scope="class") - def models(self): - return { - "orders.sql": orders_sql, - "orders_downstream.sql": orders_downstream_sql, - "schema.yml": groups_schema_yml_one_group, - } - - def test_pp_groups(self, project): - # initial run - results = run_dbt() - assert len(results) == 2 - manifest = get_manifest(project.project_root) - expected_nodes = ["model.test.orders", "model.test.orders_downstream"] - expected_groups = ["group.test.test_group"] - assert expected_nodes == sorted(list(manifest.nodes.keys())) - assert expected_groups == sorted(list(manifest.groups.keys())) - - # add group to schema - write_file(groups_schema_yml_two_groups, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - expected_nodes = ["model.test.orders", "model.test.orders_downstream"] - expected_groups = ["group.test.test_group", "group.test.test_group2"] - assert expected_nodes == sorted(list(manifest.nodes.keys())) - assert expected_groups == sorted(list(manifest.groups.keys())) - - # edit group in schema - write_file( - groups_schema_yml_two_groups_edited, project.project_root, "models", "schema.yml" - ) - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - expected_nodes = ["model.test.orders", "model.test.orders_downstream"] - expected_groups = ["group.test.test_group", "group.test.test_group2_edited"] - assert expected_nodes == sorted(list(manifest.nodes.keys())) - assert expected_groups == sorted(list(manifest.groups.keys())) - - # delete group in schema - write_file(groups_schema_yml_one_group, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - expected_nodes = ["model.test.orders", "model.test.orders_downstream"] - expected_groups = ["group.test.test_group"] - assert expected_nodes == sorted(list(manifest.nodes.keys())) - assert expected_groups == sorted(list(manifest.groups.keys())) - - # add back second group - write_file(groups_schema_yml_two_groups, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - # remove second group with model still configured to second group - write_file( - groups_schema_yml_one_group_model_in_group2, - project.project_root, - "models", - "schema.yml", - ) - with pytest.raises(ParsingError): - results = run_dbt(["--partial-parse", "run"]) - - # add back second group, make orders private with valid ref - write_file( - groups_schema_yml_two_groups_private_orders_valid_access, - project.project_root, - "models", - "schema.yml", - ) - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - write_file( - groups_schema_yml_two_groups_private_orders_invalid_access, - project.project_root, - "models", - "schema.yml", - ) - with pytest.raises(ParsingError): - results = run_dbt(["--partial-parse", "run"]) - - -my_model_c = """ -select * from {{ ref("my_model_a") }} union all -select * from {{ ref("my_model_b") }} -""" - -models_yml = """ -models: - - name: my_model_a - - name: my_model_b - - name: my_model_c -""" - -models_and_groups_yml = """ -groups: - - name: sales_analytics - owner: - name: Sales Analytics - email: sales@jaffleshop.com - -models: - - name: my_model_a - access: private - group: sales_analytics - - name: my_model_b - access: private - group: sales_analytics - - name: my_model_c - access: private - group: sales_analytics -""" - - -class TestAddingModelsToNewGroups: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_a.sql": "select 1 as id", - "my_model_b.sql": "select 2 as id", - "my_model_c.sql": my_model_c, - "models.yml": models_yml, - } - - def test_adding_models_to_new_groups(self, project): - run_dbt(["compile"]) - # This tests that the correct patch is added to my_model_c. The bug - # was that it was using the old patch, so model_c didn't have the - # correct group and access. - write_file(models_and_groups_yml, project.project_root, "models", "models.yml") - run_dbt(["compile"]) - manifest = get_manifest(project.project_root) - model_c_node = manifest.nodes["model.test.my_model_c"] - assert model_c_node.group == "sales_analytics" - assert model_c_node.access == "private" diff --git a/tests/functional/partial_parsing/test_pp_metrics.py b/tests/functional/partial_parsing/test_pp_metrics.py deleted file mode 100644 index bc688451..00000000 --- a/tests/functional/partial_parsing/test_pp_metrics.py +++ /dev/null @@ -1,85 +0,0 @@ -from dbt.tests.util import get_manifest, run_dbt, write_file -from dbt_common.exceptions import CompilationError -import pytest - -from tests.functional.partial_parsing.fixtures import ( - metric_model_a_sql, - metricflow_time_spine_sql, - people_metrics_yml, - people_metrics2_yml, - people_metrics3_yml, - people_semantic_models_yml, - people_sql, -) - - -class TestMetrics: - @pytest.fixture(scope="class") - def models(self): - return { - "people.sql": people_sql, - "metricflow_time_spine.sql": metricflow_time_spine_sql, - } - - def test_metrics(self, project): - # initial run - results = run_dbt(["run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert len(manifest.nodes) == 2 - - # Add metrics yaml file (and necessary semantic models yaml) - write_file( - people_semantic_models_yml, - project.project_root, - "models", - "people_semantic_models.yml", - ) - write_file(people_metrics_yml, project.project_root, "models", "people_metrics.yml") - results = run_dbt(["run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert len(manifest.metrics) == 2 - metric_people_id = "metric.test.number_of_people" - metric_people = manifest.metrics[metric_people_id] - expected_meta = {"my_meta": "testing"} - assert metric_people.meta == expected_meta - - # TODO: Bring back when we resolving `depends_on_nodes` - # metric_tenure_id = "metric.test.collective_tenure" - # metric_tenure = manifest.metrics[metric_tenure_id] - # assert metric_people.refs == [RefArgs(name="people")] - # assert metric_tenure.refs == [RefArgs(name="people")] - # expected_depends_on_nodes = ["model.test.people"] - # assert metric_people.depends_on.nodes == expected_depends_on_nodes - - # Change metrics yaml files - write_file(people_metrics2_yml, project.project_root, "models", "people_metrics.yml") - results = run_dbt(["run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - metric_people = manifest.metrics[metric_people_id] - expected_meta = {"my_meta": "replaced"} - assert metric_people.meta == expected_meta - # TODO: Bring back when we resolving `depends_on_nodes` - # expected_depends_on_nodes = ["model.test.people"] - # assert metric_people.depends_on.nodes == expected_depends_on_nodes - - # Add model referring to metric - write_file(metric_model_a_sql, project.project_root, "models", "metric_model_a.sql") - results = run_dbt(["run"]) - manifest = get_manifest(project.project_root) - # TODO: Bring back when we resolving `depends_on_nodes` - # model_a = manifest.nodes["model.test.metric_model_a"] - # expected_depends_on_nodes = [ - # "metric.test.number_of_people", - # "metric.test.collective_tenure", - # ] - # assert model_a.depends_on.nodes == expected_depends_on_nodes - - # Then delete a metric - write_file(people_metrics3_yml, project.project_root, "models", "people_metrics.yml") - with pytest.raises(CompilationError): - # We use "parse" here and not "run" because we're checking that the CompilationError - # occurs at parse time, not compilation - results = run_dbt(["parse"]) diff --git a/tests/functional/partial_parsing/test_pp_vars.py b/tests/functional/partial_parsing/test_pp_vars.py deleted file mode 100644 index c903cdea..00000000 --- a/tests/functional/partial_parsing/test_pp_vars.py +++ /dev/null @@ -1,398 +0,0 @@ -import os -from pathlib import Path - -from dbt.adapters.exceptions import FailedToConnectError -from dbt.constants import SECRET_ENV_PREFIX -from dbt.exceptions import ParsingError -from dbt.tests.util import get_manifest, write_file -import pytest - -from tests.functional.partial_parsing.fixtures import ( - env_var_macro_sql, - env_var_macros_yml, - env_var_metrics_yml, - env_var_model_one_sql, - env_var_model_sql, - env_var_model_test_yml, - env_var_schema_yml, - env_var_schema2_yml, - env_var_schema3_yml, - env_var_sources_yml, - metricflow_time_spine_sql, - model_color_sql, - model_one_sql, - people_semantic_models_yml, - people_sql, - raw_customers_csv, - test_color_sql, -) -from tests.functional.utils import run_dbt, run_dbt_and_capture - - -os.environ["DBT_PP_TEST"] = "true" - - -class TestEnvVars: - @pytest.fixture(scope="class") - def models(self): - return { - "model_color.sql": model_color_sql, - } - - def test_env_vars_models(self, project): - # initial run - results = run_dbt(["run"]) - assert len(results) == 1 - - # copy a file with an env_var call without an env_var - write_file(env_var_model_sql, project.project_root, "models", "env_var_model.sql") - with pytest.raises(ParsingError): - results = run_dbt(["--partial-parse", "run"]) - - # set the env var - os.environ["ENV_VAR_TEST"] = "TestingEnvVars" - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - expected_env_vars = {"ENV_VAR_TEST": "TestingEnvVars"} - assert expected_env_vars == manifest.env_vars - model_id = "model.test.env_var_model" - model = manifest.nodes[model_id] - model_created_at = model.created_at - - # change the env var - os.environ["ENV_VAR_TEST"] = "second" - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - expected_env_vars = {"ENV_VAR_TEST": "second"} - assert expected_env_vars == manifest.env_vars - assert model_created_at != manifest.nodes[model_id].created_at - - # set an env_var in a schema file - write_file(env_var_schema_yml, project.project_root, "models", "schema.yml") - write_file(env_var_model_one_sql, project.project_root, "models", "model_one.sql") - with pytest.raises(ParsingError): - results = run_dbt(["--partial-parse", "run"]) - - # actually set the env_var - os.environ["TEST_SCHEMA_VAR"] = "view" - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - expected_env_vars = {"ENV_VAR_TEST": "second", "TEST_SCHEMA_VAR": "view"} - assert expected_env_vars == manifest.env_vars - - # env vars in a source - os.environ["ENV_VAR_DATABASE"] = "dbt" - os.environ["ENV_VAR_SEVERITY"] = "warn" - write_file(raw_customers_csv, project.project_root, "seeds", "raw_customers.csv") - write_file(env_var_sources_yml, project.project_root, "models", "sources.yml") - run_dbt(["--partial-parse", "seed"]) - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - manifest = get_manifest(project.project_root) - expected_env_vars = { - "ENV_VAR_TEST": "second", - "TEST_SCHEMA_VAR": "view", - "ENV_VAR_DATABASE": "dbt", - "ENV_VAR_SEVERITY": "warn", - } - assert expected_env_vars == manifest.env_vars - assert len(manifest.sources) == 1 - source_id = "source.test.seed_sources.raw_customers" - source = manifest.sources[source_id] - assert source.database == "dbt" - schema_file = manifest.files[source.file_id] - test_id = "test.test.source_not_null_seed_sources_raw_customers_id.e39ee7bf0d" - test_node = manifest.nodes[test_id] - assert test_node.config.severity == "WARN" - - # Change severity env var - os.environ["ENV_VAR_SEVERITY"] = "error" - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - expected_env_vars = { - "ENV_VAR_TEST": "second", - "TEST_SCHEMA_VAR": "view", - "ENV_VAR_DATABASE": "dbt", - "ENV_VAR_SEVERITY": "error", - } - assert expected_env_vars == manifest.env_vars - source_id = "source.test.seed_sources.raw_customers" - source = manifest.sources[source_id] - schema_file = manifest.files[source.file_id] - expected_schema_file_env_vars = { - "sources": {"seed_sources": ["ENV_VAR_DATABASE", "ENV_VAR_SEVERITY"]} - } - assert expected_schema_file_env_vars == schema_file.env_vars - test_node = manifest.nodes[test_id] - assert test_node.config.severity == "ERROR" - - # Change database env var - os.environ["ENV_VAR_DATABASE"] = "test_dbt" - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - expected_env_vars = { - "ENV_VAR_TEST": "second", - "TEST_SCHEMA_VAR": "view", - "ENV_VAR_DATABASE": "test_dbt", - "ENV_VAR_SEVERITY": "error", - } - assert expected_env_vars == manifest.env_vars - source = manifest.sources[source_id] - assert source.database == "test_dbt" - - # Delete database env var - del os.environ["ENV_VAR_DATABASE"] - with pytest.raises(ParsingError): - results = run_dbt(["--partial-parse", "run"]) - os.environ["ENV_VAR_DATABASE"] = "test_dbt" - - # Add generic test with test kwarg that's rendered late (no curly brackets) - os.environ["ENV_VAR_DATABASE"] = "dbt" - write_file(test_color_sql, project.project_root, "macros", "test_color.sql") - results = run_dbt(["--partial-parse", "run"]) - # Add source test using test_color and an env_var for color - write_file(env_var_schema2_yml, project.project_root, "models/schema.yml") - with pytest.raises(ParsingError): - results = run_dbt(["--partial-parse", "run"]) - os.environ["ENV_VAR_COLOR"] = "green" - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - test_color_id = "test.test.check_color_model_one_env_var_ENV_VAR_COLOR___fun.89638de387" - test_node = manifest.nodes[test_color_id] - # kwarg was rendered but not changed (it will be rendered again when compiled) - assert test_node.test_metadata.kwargs["color"] == "env_var('ENV_VAR_COLOR')" - results = run_dbt(["--partial-parse", "test"]) - - # Add an exposure with an env_var - os.environ["ENV_VAR_OWNER"] = "John Doe" - write_file(env_var_schema3_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - expected_env_vars = { - "ENV_VAR_TEST": "second", - "TEST_SCHEMA_VAR": "view", - "ENV_VAR_DATABASE": "dbt", - "ENV_VAR_SEVERITY": "error", - "ENV_VAR_COLOR": "green", - "ENV_VAR_OWNER": "John Doe", - } - assert expected_env_vars == manifest.env_vars - exposure = list(manifest.exposures.values())[0] - schema_file = manifest.files[exposure.file_id] - expected_sf_env_vars = { - "models": {"model_one": ["TEST_SCHEMA_VAR", "ENV_VAR_COLOR"]}, - "exposures": {"proxy_for_dashboard": ["ENV_VAR_OWNER"]}, - } - assert expected_sf_env_vars == schema_file.env_vars - - # add a macro and a macro schema file - os.environ["ENV_VAR_SOME_KEY"] = "toodles" - write_file(env_var_macro_sql, project.project_root, "macros", "env_var_macro.sql") - write_file(env_var_macros_yml, project.project_root, "macros", "env_var_macros.yml") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - expected_env_vars = { - "ENV_VAR_TEST": "second", - "TEST_SCHEMA_VAR": "view", - "ENV_VAR_DATABASE": "dbt", - "ENV_VAR_SEVERITY": "error", - "ENV_VAR_COLOR": "green", - "ENV_VAR_OWNER": "John Doe", - "ENV_VAR_SOME_KEY": "toodles", - } - assert expected_env_vars == manifest.env_vars - macro_id = "macro.test.do_something" - macro = manifest.macros[macro_id] - assert macro.meta == {"some_key": "toodles"} - # change the env var - os.environ["ENV_VAR_SOME_KEY"] = "dumdedum" - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - macro = manifest.macros[macro_id] - assert macro.meta == {"some_key": "dumdedum"} - - # Add a schema file with a test on model_color and env_var in test enabled config - write_file(env_var_model_test_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - manifest = get_manifest(project.project_root) - model_color = manifest.nodes["model.test.model_color"] - schema_file = manifest.files[model_color.patch_path] - expected_env_vars = { - "models": { - "model_one": ["TEST_SCHEMA_VAR", "ENV_VAR_COLOR"], - "model_color": ["ENV_VAR_ENABLED"], - }, - "exposures": {"proxy_for_dashboard": ["ENV_VAR_OWNER"]}, - } - assert expected_env_vars == schema_file.env_vars - - # Add a metrics file with env_vars - os.environ["ENV_VAR_METRICS"] = "TeStInG" - write_file(people_sql, project.project_root, "models", "people.sql") - write_file( - metricflow_time_spine_sql, project.project_root, "models", "metricflow_time_spine.sql" - ) - write_file( - people_semantic_models_yml, project.project_root, "models", "semantic_models.yml" - ) - write_file(env_var_metrics_yml, project.project_root, "models", "metrics.yml") - results = run_dbt(["run"]) - manifest = get_manifest(project.project_root) - assert "ENV_VAR_METRICS" in manifest.env_vars - assert manifest.env_vars["ENV_VAR_METRICS"] == "TeStInG" - metric_node = manifest.metrics["metric.test.number_of_people"] - assert metric_node.meta == {"my_meta": "TeStInG"} - - # Change metrics env var - os.environ["ENV_VAR_METRICS"] = "Changed!" - results = run_dbt(["run"]) - manifest = get_manifest(project.project_root) - metric_node = manifest.metrics["metric.test.number_of_people"] - assert metric_node.meta == {"my_meta": "Changed!"} - - # delete the env vars to cleanup - del os.environ["ENV_VAR_TEST"] - del os.environ["ENV_VAR_SEVERITY"] - del os.environ["ENV_VAR_DATABASE"] - del os.environ["TEST_SCHEMA_VAR"] - del os.environ["ENV_VAR_COLOR"] - del os.environ["ENV_VAR_SOME_KEY"] - del os.environ["ENV_VAR_OWNER"] - del os.environ["ENV_VAR_METRICS"] - - -class TestProjectEnvVars: - @pytest.fixture(scope="class") - def project_config_update(self): - # Need to set the environment variable here initially because - # the project fixture loads the config. - os.environ["ENV_VAR_NAME"] = "Jane Smith" - return {"models": {"+meta": {"meta_name": "{{ env_var('ENV_VAR_NAME') }}"}}} - - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - def test_project_env_vars(self, project): - # Initial run - results = run_dbt(["run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - state_check = manifest.state_check - model_id = "model.test.model_one" - model = manifest.nodes[model_id] - assert model.config.meta["meta_name"] == "Jane Smith" - env_vars_hash_checksum = state_check.project_env_vars_hash.checksum - - # Change the environment variable - os.environ["ENV_VAR_NAME"] = "Jane Doe" - results = run_dbt(["run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - model = manifest.nodes[model_id] - assert model.config.meta["meta_name"] == "Jane Doe" - assert env_vars_hash_checksum != manifest.state_check.project_env_vars_hash.checksum - - # cleanup - del os.environ["ENV_VAR_NAME"] - - -class TestProfileEnvVars: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - @pytest.fixture(scope="class") - def dbt_profile_target(self): - # Need to set these here because the base integration test class - # calls 'load_config' before the tests are run. - # Note: only the specified profile is rendered, so there's no - # point it setting env_vars in non-used profiles. - os.environ["ENV_VAR_USER"] = "root" - os.environ["ENV_VAR_PASS"] = "password" - return { - "type": "postgres", - "threads": 4, - "host": "localhost", - "port": 5432, - "user": "{{ env_var('ENV_VAR_USER') }}", - "pass": "{{ env_var('ENV_VAR_PASS') }}", - "dbname": "dbt", - } - - def test_profile_env_vars(self, project, logs_dir): - # Initial run - os.environ["ENV_VAR_USER"] = "root" - os.environ["ENV_VAR_PASS"] = "password" - - run_dbt(["run"]) - manifest = get_manifest(project.project_root) - env_vars_checksum = manifest.state_check.profile_env_vars_hash.checksum - - # Change env_vars, the user doesn't exist, this should fail - os.environ["ENV_VAR_USER"] = "fake_user" - - # N.B. run_dbt_and_capture won't work here because FailedToConnectError ends the test entirely - with pytest.raises(FailedToConnectError): - run_dbt(["run"], expect_pass=False) - - log_output = Path(logs_dir, "dbt.log").read_text() - assert "env vars used in profiles.yml have changed" in log_output - - manifest = get_manifest(project.project_root) - assert env_vars_checksum != manifest.state_check.profile_env_vars_hash.checksum - - -class TestProfileSecretEnvVars: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - @property - def dbt_profile_target(self): - # Need to set these here because the base integration test class - # calls 'load_config' before the tests are run. - # Note: only the specified profile is rendered, so there's no - # point in setting env_vars in non-used profiles. - - # user is secret and password is not. postgres on macos doesn't care if the password - # changes so we have to change the user. related: https://github.com/dbt-labs/dbt-core/pull/4250 - os.environ[SECRET_ENV_PREFIX + "USER"] = "root" - os.environ["ENV_VAR_PASS"] = "password" - return { - "type": "postgres", - "threads": 4, - "host": "localhost", - "port": 5432, - "user": "{{ env_var('DBT_ENV_SECRET_USER') }}", - "pass": "{{ env_var('ENV_VAR_PASS') }}", - "dbname": "dbt", - } - - def test_profile_secret_env_vars(self, project): - # Initial run - os.environ[SECRET_ENV_PREFIX + "USER"] = "root" - os.environ["ENV_VAR_PASS"] = "password" - - results = run_dbt(["run"]) - manifest = get_manifest(project.project_root) - env_vars_checksum = manifest.state_check.profile_env_vars_hash.checksum - - # Change a secret var, it shouldn't register because we shouldn't save secrets. - os.environ[SECRET_ENV_PREFIX + "USER"] = "fake_user" - # we just want to see if the manifest has included - # the secret in the hash of environment variables. - (results, log_output) = run_dbt_and_capture(["run"], expect_pass=True) - # I020 is the event code for "env vars used in profiles.yml have changed" - assert not ("I020" in log_output) - manifest = get_manifest(project.project_root) - assert env_vars_checksum == manifest.state_check.profile_env_vars_hash.checksum diff --git a/tests/functional/partial_parsing/test_versioned_models.py b/tests/functional/partial_parsing/test_versioned_models.py deleted file mode 100644 index d725c671..00000000 --- a/tests/functional/partial_parsing/test_versioned_models.py +++ /dev/null @@ -1,128 +0,0 @@ -import pathlib - -from dbt.exceptions import DuplicateVersionedUnversionedError -from dbt.tests.util import ( - get_manifest, - read_file, - rm_file, - run_dbt, - write_file, -) -import pytest - - -model_one_sql = """ -select 1 as fun -""" - -model_one_downstream_sql = """ -select fun from {{ ref('model_one') }} -""" - -models_versions_schema_yml = """ - -models: - - name: model_one - description: "The first model" - versions: - - v: 1 - - v: 2 -""" - -models_versions_defined_in_schema_yml = """ -models: - - name: model_one - description: "The first model" - versions: - - v: 1 - - v: 2 - defined_in: model_one_different -""" - -models_versions_updated_schema_yml = """ -models: - - name: model_one - latest_version: 1 - description: "The first model" - versions: - - v: 1 - - v: 2 - defined_in: model_one_different -""" - -model_two_sql = """ -select 1 as notfun -""" - - -class TestVersionedModels: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one_v1.sql": model_one_sql, - "model_one.sql": model_one_sql, - "model_one_downstream.sql": model_one_downstream_sql, - "schema.yml": models_versions_schema_yml, - } - - def test_pp_versioned_models(self, project): - results = run_dbt(["run"]) - assert len(results) == 3 - - manifest = get_manifest(project.project_root) - model_one_node = manifest.nodes["model.test.model_one.v1"] - assert not model_one_node.is_latest_version - model_two_node = manifest.nodes["model.test.model_one.v2"] - assert model_two_node.is_latest_version - # assert unpinned ref points to latest version - model_one_downstream_node = manifest.nodes["model.test.model_one_downstream"] - assert model_one_downstream_node.depends_on.nodes == ["model.test.model_one.v2"] - - # update schema.yml block - model_one is now 'defined_in: model_one_different' - rm_file(project.project_root, "models", "model_one.sql") - write_file(model_one_sql, project.project_root, "models", "model_one_different.sql") - write_file( - models_versions_defined_in_schema_yml, project.project_root, "models", "schema.yml" - ) - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # update versions schema.yml block - latest_version from 2 to 1 - write_file( - models_versions_updated_schema_yml, project.project_root, "models", "schema.yml" - ) - # This is where the test was failings in a CI run with: - # relation \"test..._test_partial_parsing.model_one_downstream\" does not exist - # because in core/dbt/include/global_project/macros/materializations/models/view/view.sql - # "existing_relation" didn't actually exist by the time it gets to the rename of the - # existing relation. - (pathlib.Path(project.project_root) / "log_output").mkdir(parents=True, exist_ok=True) - results = run_dbt( - ["--partial-parse", "--log-format-file", "json", "--log-path", "log_output", "run"] - ) - assert len(results) == 3 - - manifest = get_manifest(project.project_root) - model_one_node = manifest.nodes["model.test.model_one.v1"] - assert model_one_node.is_latest_version - model_two_node = manifest.nodes["model.test.model_one.v2"] - assert not model_two_node.is_latest_version - # assert unpinned ref points to latest version - model_one_downstream_node = manifest.nodes["model.test.model_one_downstream"] - assert model_one_downstream_node.depends_on.nodes == ["model.test.model_one.v1"] - - # assert unpinned ref to latest-not-max version yields an "FYI" info-level log - log_output = read_file("log_output", "dbt.log").replace("\n", " ").replace("\\n", " ") - assert "UnpinnedRefNewVersionAvailable" in log_output - - # update versioned model - write_file(model_two_sql, project.project_root, "models", "model_one_different.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - manifest = get_manifest(project.project_root) - assert len(manifest.nodes) == 3 - - # create a new model_one in model_one.sql and re-parse - write_file(model_one_sql, project.project_root, "models", "model_one.sql") - with pytest.raises(DuplicateVersionedUnversionedError): - run_dbt(["parse"]) From f67e02f80d9c8d76fac262fea1b41cf77342cb75 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 16 Apr 2024 15:05:38 -0400 Subject: [PATCH 2/5] Fix `psycopg2` changelog entry (#61) --- .changes/unreleased/Fixes-20240412-153154.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Fixes-20240412-153154.yaml b/.changes/unreleased/Fixes-20240412-153154.yaml index 10bac271..c64659af 100644 --- a/.changes/unreleased/Fixes-20240412-153154.yaml +++ b/.changes/unreleased/Fixes-20240412-153154.yaml @@ -1,5 +1,5 @@ kind: Fixes -body: Determine `psycopg2` based on `platform_system` (Linux or other) +body: Determine `psycopg2` based on `platform_system` (Linux or other), remove usage of `DBT_PSYCOPG2_NAME` environment variable time: 2024-04-12T15:31:54.861201-04:00 custom: Author: mikealfare From 3e7940a1160e1af0fd1c2dcba6c8723cf22055eb Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:11:59 -0500 Subject: [PATCH 3/5] manually update version, do changie batch and merge (#63) --- .changes/1.0.8-b3.md | 9 +++++++++ .changes/unreleased/Fixes-20240412-153154.yaml | 6 ------ .../unreleased/Under the Hood-20240410-180644.yaml | 6 ------ CHANGELOG.md | 11 +++++++++++ dbt/adapters/postgres/__version__.py | 2 +- 5 files changed, 21 insertions(+), 13 deletions(-) create mode 100644 .changes/1.0.8-b3.md delete mode 100644 .changes/unreleased/Fixes-20240412-153154.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240410-180644.yaml diff --git a/.changes/1.0.8-b3.md b/.changes/1.0.8-b3.md new file mode 100644 index 00000000..d73520b5 --- /dev/null +++ b/.changes/1.0.8-b3.md @@ -0,0 +1,9 @@ +## dbt-postgres 1.0.8-b3 - April 16, 2024 + +### Fixes + +* Determine `psycopg2` based on `platform_system` (Linux or other), remove usage of `DBT_PSYCOPG2_NAME` environment variable + +### Under the Hood + +* Update dependabot configuration to cover GHA diff --git a/.changes/unreleased/Fixes-20240412-153154.yaml b/.changes/unreleased/Fixes-20240412-153154.yaml deleted file mode 100644 index c64659af..00000000 --- a/.changes/unreleased/Fixes-20240412-153154.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Determine `psycopg2` based on `platform_system` (Linux or other), remove usage of `DBT_PSYCOPG2_NAME` environment variable -time: 2024-04-12T15:31:54.861201-04:00 -custom: - Author: mikealfare - Issue: "60" diff --git a/.changes/unreleased/Under the Hood-20240410-180644.yaml b/.changes/unreleased/Under the Hood-20240410-180644.yaml deleted file mode 100644 index 2f7eeda6..00000000 --- a/.changes/unreleased/Under the Hood-20240410-180644.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Update dependabot configuration to cover GHA -time: 2024-04-10T18:06:44.884603-04:00 -custom: - Author: mikealfare - Issue: "56" diff --git a/CHANGELOG.md b/CHANGELOG.md index 29a31234..6c13880e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,3 +19,14 @@ and is generated by [Changie](https://github.com/miniscruff/changie). ### Security * Pin `black>=24.3` in `pyproject.toml` + +## dbt-postgres 1.0.8-b3 - April 16, 2024 + +### Fixes + +* Determine `psycopg2` based on `platform_system` (Linux or other), remove usage of `DBT_PSYCOPG2_NAME` environment variable + +### Under the Hood + +* Update dependabot configuration to cover GHA + diff --git a/dbt/adapters/postgres/__version__.py b/dbt/adapters/postgres/__version__.py index 7d16c28f..b0f82cbc 100644 --- a/dbt/adapters/postgres/__version__.py +++ b/dbt/adapters/postgres/__version__.py @@ -1 +1 @@ -version = "1.8.0b2" +version = "1.8.0b3" From f40554d7fa2132ee731ee8df7e0c485e76027d3c Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Tue, 16 Apr 2024 18:42:41 -0500 Subject: [PATCH 4/5] bumping to beta4 due to issue during release (#65) --- dbt/adapters/postgres/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/postgres/__version__.py b/dbt/adapters/postgres/__version__.py index b0f82cbc..6b76061f 100644 --- a/dbt/adapters/postgres/__version__.py +++ b/dbt/adapters/postgres/__version__.py @@ -1 +1 @@ -version = "1.8.0b3" +version = "1.8.0b4" From 333a1e59bed45dc2c42feab409154791812b1c99 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 19 Apr 2024 13:19:08 -0400 Subject: [PATCH 5/5] Add and configure `pre-commit` (#62) --- .github/workflows/code-quality.yml | 28 +- .github/workflows/integration-tests.yml | 9 - .github/workflows/unit-tests.yml | 8 - .pre-commit-config.yaml | 55 +++ CHANGELOG.md | 1 - CONTRIBUTING.md | 2 +- dbt/adapters/postgres/relation.py | 30 +- .../postgres/relation_configs/index.py | 2 +- pyproject.toml | 72 +--- tests/functional/artifacts/test_artifacts.py | 11 - .../artifacts/test_previous_version_state.py | 2 +- .../context_methods/test_builtin_functions.py | 1 - tests/functional/dbt_debug/test_dbt_debug.py | 5 +- tests/functional/dbt_runner.py | 4 +- tests/functional/profiles/test_profile_dir.py | 168 -------- .../functional/profiles/test_profiles_yml.py | 65 --- .../schema_tests/test_schema_v2_tests.py | 4 +- .../shared_tests/test_hooks/test_hooks.py | 1 + .../test_simple_seed/test_simple_seed.py | 1 + tests/functional/source_overrides/fixtures.py | 387 ------------------ .../test_simple_source_override.py | 146 ------- .../test_source_overrides_duplicate_model.py | 68 --- tests/functional/test_connection_manager.py | 7 +- tests/functional/test_dbt_runner.py | 14 +- tests/functional/test_init.py | 1 - tests/unit/test_connection.py | 6 +- tests/unit/test_renamed_relations.py | 12 +- 27 files changed, 121 insertions(+), 989 deletions(-) create mode 100644 .pre-commit-config.yaml delete mode 100644 tests/functional/profiles/test_profile_dir.py delete mode 100644 tests/functional/profiles/test_profiles_yml.py delete mode 100644 tests/functional/source_overrides/fixtures.py delete mode 100644 tests/functional/source_overrides/test_simple_source_override.py delete mode 100644 tests/functional/source_overrides/test_source_overrides_duplicate_model.py diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 081079fc..1ca2ded0 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -9,16 +9,14 @@ on: workflow_dispatch: inputs: dbt_adapters_branch: - description: "The branch of dbt-adapters to evaluate" - type: string - required: true - default: "main" + description: "The branch of dbt-adapters to evaluate" + type: string + default: "main" workflow_call: inputs: dbt_adapters_branch: description: "The branch of dbt-adapters to evaluate" type: string - required: true default: "main" permissions: read-all @@ -27,16 +25,12 @@ permissions: read-all concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} cancel-in-progress: true -env: - # set DBT_ADAPTERS_BRANCH to the input value if the event is a workflow_dispatch (workflow_call uses the same event_name), - # otherwise use 'main' - DBT_ADAPTERS_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.dbt_adapters_branch || 'main' }} + jobs: code-quality: name: Code Quality runs-on: ubuntu-latest - steps: - name: Check out repository uses: actions/checkout@v4 @@ -44,17 +38,13 @@ jobs: persist-credentials: false - name: Update Adapters and Core branches + if: ${{ contains(github.event_name, 'workflow_') }} shell: bash - run: | - ./.github/scripts/update_dev_packages.sh \ - $DBT_ADAPTERS_BRANCH \ - "main" + run: ./.github/scripts/update_dev_packages.sh ${{ inputs.dbt_adapters_branch }} "main" - name: Setup `hatch` uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - - name: Run linters - run: hatch run lint:all - - - name: Run typechecks - run: hatch run typecheck:all + - name: Run code quality + shell: bash + run: hatch run code-quality diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 6bb37ec5..4bb423fa 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -31,7 +31,6 @@ on: required: false default: "main" - permissions: read-all # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise @@ -101,14 +100,6 @@ jobs: POSTGRES_TEST_DATABASE: dbt POSTGRES_TEST_THREADS: 4 - - name: Publish results - uses: dbt-labs/dbt-adapters/.github/actions/publish-results@main - if: always() - with: - source-file: "results.csv" - file-name: "integration_results" - python-version: ${{ matrix.python-version }} - psycopg2-check: name: "Test psycopg2 build version" runs-on: ${{ matrix.scenario.platform }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 69681c0f..91d8c578 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -39,11 +39,3 @@ jobs: - name: Run unit tests run: hatch run unit-tests:all shell: bash - - - name: Publish results - uses: dbt-labs/dbt-adapters/.github/actions/publish-results@main - if: always() - with: - source-file: "results.csv" - file-name: "unit_results" - python-version: ${{ matrix.python-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..c869fe86 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,55 @@ +default_language_version: + python: python3 + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml + args: [--unsafe] + - id: check-json + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-case-conflict + +- repo: https://github.com/dbt-labs/pre-commit-hooks + rev: v0.1.0a1 + hooks: + - id: dbt-core-in-adapters-check + +- repo: https://github.com/psf/black + rev: 24.4.0 + hooks: + - id: black + args: + - --line-length=99 + - --target-version=py38 + - --target-version=py39 + - --target-version=py310 + - --target-version=py311 + +- repo: https://github.com/pycqa/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + exclude: tests/ + args: + - --max-line-length=99 + - --select=E,F,W + - --ignore=E203,E501,E741,W503,W504 + - --per-file-ignores=*/__init__.py:F401 + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.9.0 + hooks: + - id: mypy + args: + - --explicit-package-bases + - --ignore-missing-imports + - --pretty + - --show-error-codes + files: ^dbt/adapters/postgres + additional_dependencies: + - types-PyYAML + - types-protobuf + - types-pytz diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c13880e..f9ef4153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,4 +29,3 @@ and is generated by [Changie](https://github.com/miniscruff/changie). ### Under the Hood * Update dependabot configuration to cover GHA - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f39e4cb5..427d2de5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -164,7 +164,7 @@ Remember to commit and push the file that's created. ### Signing the CLA -> **_NOTE:_** All contributors to `dbt-postgres` must sign the +> **_NOTE:_** All contributors to `dbt-postgres` must sign the > [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements)(CLA). Maintainers will be unable to merge contributions until the contributor signs the CLA. diff --git a/dbt/adapters/postgres/relation.py b/dbt/adapters/postgres/relation.py index 677b12ac..05d55237 100644 --- a/dbt/adapters/postgres/relation.py +++ b/dbt/adapters/postgres/relation.py @@ -20,19 +20,23 @@ @dataclass(frozen=True, eq=False, repr=False) class PostgresRelation(BaseRelation): - renameable_relations: FrozenSet[RelationType] = field(default_factory=lambda: frozenset( - { - RelationType.View, - RelationType.Table, - RelationType.MaterializedView, - } - )) - replaceable_relations: FrozenSet[RelationType] = field(default_factory=lambda: frozenset( - { - RelationType.View, - RelationType.Table, - } - )) + renameable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset( + { + RelationType.View, + RelationType.Table, + RelationType.MaterializedView, + } + ) + ) + replaceable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset( + { + RelationType.View, + RelationType.Table, + } + ) + ) def __post_init__(self): # Check for length of Postgres table/view names. diff --git a/dbt/adapters/postgres/relation_configs/index.py b/dbt/adapters/postgres/relation_configs/index.py index c4863073..0ed3e8aa 100644 --- a/dbt/adapters/postgres/relation_configs/index.py +++ b/dbt/adapters/postgres/relation_configs/index.py @@ -23,7 +23,7 @@ class PostgresIndexMethod(StrEnum): @classmethod def default(cls) -> "PostgresIndexMethod": - return cls.btree + return cls("btree") @dataclass(frozen=True, eq=True, unsafe_hash=True) diff --git a/pyproject.toml b/pyproject.toml index 996ea46a..0e93423c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,56 +59,30 @@ path = "dbt/adapters/postgres/__version__.py" dependencies = [ "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + 'pre-commit==3.7.0;python_version>="3.9"', + 'pre-commit==3.5.0;python_version=="3.8"', ] - -[tool.hatch.envs.lint] -detached = true -dependencies = [ - "black>=24.3", - "flake8", - "Flake8-pyproject", -] -[tool.hatch.envs.lint.scripts] -all = [ - "black", - "flake8", -] -black = "python -m black ." -flake8 = "python -m flake8 ." - -[tool.hatch.envs.typecheck] -dependencies = [ - "mypy", - "types-protobuf", - "types-pytz", -] -[tool.hatch.envs.typecheck.scripts] -all = "python -m mypy ." +[tool.hatch.envs.default.scripts] +dev = "pre-commit install" +code-quality = "pre-commit run --all-files" [tool.hatch.envs.unit-tests] dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", "freezegun", "pytest", "pytest-dotenv", "pytest-mock", "pytest-xdist", ] -extra-dependencies = [ - # TODO: remove `dbt-core` dependencies from unit tests - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", - "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", - "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", -] [tool.hatch.envs.unit-tests.scripts] all = "python -m pytest {args:tests/unit}" [tool.hatch.envs.integration-tests] template = "unit-tests" extra-dependencies = [ - # TODO: remove `dbt-core` dependencies from integration tests - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", - "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", - "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", ] [tool.hatch.envs.integration-tests.env-vars] @@ -125,8 +99,6 @@ dependencies = [ "twine", "check-wheel-contents", ] - - [tool.hatch.envs.build.scripts] check-all = [ "- check-wheel", @@ -143,35 +115,9 @@ check-sdist = [ "pip freeze | grep dbt-postgres", ] -[tool.black] -line-length = 99 -target-version = ['py38'] - -[tool.flake8] -select = ["E", "W", "F"] -ignore = ["E203", "E501", "E741", "W503", "W504"] -exclude = ["tests", "venv", ".hatch_venvs"] -per-file-ignores = ["*/__init__.py: F401"] - -[tool.mypy] -namespace_packages = true -show_error_codes = true -explicit_package_bases = true -ignore_missing_imports = true -pretty = true -files = [ - "dbt/adapters/postgres", - "tests/unit", -] -exclude = [ - "tests/functional", - "venv", - ".hatch_venvs", -] - [tool.pytest] env_files = ["test.env"] testpaths = [ "tests/functional", "tests/unit", -] \ No newline at end of file +] diff --git a/tests/functional/artifacts/test_artifacts.py b/tests/functional/artifacts/test_artifacts.py index 756ee73f..ca2c03f0 100644 --- a/tests/functional/artifacts/test_artifacts.py +++ b/tests/functional/artifacts/test_artifacts.py @@ -1,4 +1,3 @@ -from datetime import datetime import jsonschema import os @@ -12,16 +11,6 @@ ) import pytest -from tests.functional.artifacts.expected_manifest import ( - expected_references_manifest, - expected_seeded_manifest, - expected_versions_manifest, -) -from tests.functional.artifacts.expected_run_results import ( - expected_references_run_results, - expected_run_results, - expected_versions_run_results, -) from tests.functional.utils import run_dbt, run_dbt_and_capture diff --git a/tests/functional/artifacts/test_previous_version_state.py b/tests/functional/artifacts/test_previous_version_state.py index a19a2486..5ed37b9a 100644 --- a/tests/functional/artifacts/test_previous_version_state.py +++ b/tests/functional/artifacts/test_previous_version_state.py @@ -5,7 +5,7 @@ from dbt.artifacts.schemas.base import get_artifact_schema_version from dbt.artifacts.schemas.run import RunResultsArtifact from dbt.contracts.graph.manifest import WritableManifest -from dbt.exceptions import IncompatibleSchemaError +from dbt.artifacts.exceptions import IncompatibleSchemaError from dbt.tests.util import get_manifest import pytest diff --git a/tests/functional/context_methods/test_builtin_functions.py b/tests/functional/context_methods/test_builtin_functions.py index f5d7b92f..b8a47b34 100644 --- a/tests/functional/context_methods/test_builtin_functions.py +++ b/tests/functional/context_methods/test_builtin_functions.py @@ -1,5 +1,4 @@ import json -import os from dbt.tests.util import write_file from dbt_common.exceptions import CompilationError diff --git a/tests/functional/dbt_debug/test_dbt_debug.py b/tests/functional/dbt_debug/test_dbt_debug.py index 3e2a182e..e318322e 100644 --- a/tests/functional/dbt_debug/test_dbt_debug.py +++ b/tests/functional/dbt_debug/test_dbt_debug.py @@ -1,9 +1,6 @@ import pytest -import os -import re -import yaml -from tests.functional.utils import run_dbt, run_dbt_and_capture +from tests.functional.utils import run_dbt_and_capture MODELS__MODEL_SQL = """ seled 1 as id diff --git a/tests/functional/dbt_runner.py b/tests/functional/dbt_runner.py index 01ebc873..ba7aa2ba 100644 --- a/tests/functional/dbt_runner.py +++ b/tests/functional/dbt_runner.py @@ -25,13 +25,15 @@ def assert_run_results_have_compiled_node_attributes( class dbtTestRunner(dbtRunner): + exit_assertions: List[Callable[[List[str], dbtRunnerResult], None]] + def __init__( self, manifest: Optional[Manifest] = None, callbacks: Optional[List[Callable[[EventMsg], None]]] = None, exit_assertions: Optional[List[Callable[[List[str], dbtRunnerResult], None]]] = None, ): - self.exit_assertions = exit_assertions if exit_assertions else _STANDARD_ASSERTIONS + self.exit_assertions = exit_assertions if exit_assertions else _STANDARD_ASSERTIONS # type: ignore super().__init__(manifest, callbacks) def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult: diff --git a/tests/functional/profiles/test_profile_dir.py b/tests/functional/profiles/test_profile_dir.py deleted file mode 100644 index 282c978c..00000000 --- a/tests/functional/profiles/test_profile_dir.py +++ /dev/null @@ -1,168 +0,0 @@ -from contextlib import contextmanager -import os -from pathlib import Path - -from dbt.tests.util import rm_file, write_file -import pytest -import yaml - -from tests.functional.utils import run_dbt, run_dbt_and_capture - - -@pytest.fixture(scope="class") -def profiles_yml(profiles_root, dbt_profile_data): - write_file(yaml.safe_dump(dbt_profile_data), profiles_root, "profiles.yml") - return dbt_profile_data - - -@pytest.fixture(scope="class") -def profiles_home_root(): - return os.path.join(os.path.expanduser("~"), ".dbt") - - -@pytest.fixture(scope="class") -def profiles_env_root(tmpdir_factory): - path = tmpdir_factory.mktemp("profile_env") - # environment variables are lowercased for some reason in _get_flag_value_from_env within dbt.flags - return str(path).lower() - - -@pytest.fixture(scope="class") -def profiles_flag_root(tmpdir_factory): - return tmpdir_factory.mktemp("profile_flag") - - -@pytest.fixture(scope="class") -def profiles_project_root(project): - return project.project_root - - -@pytest.fixture(scope="class") -def cwd(): - return os.getcwd() - - -@pytest.fixture(scope="class") -def cwd_parent(cwd): - return os.path.dirname(cwd) - - -@pytest.fixture(scope="class") -def cwd_child(): - # pick any child directory of the dbt project - return Path(os.getcwd()) / "macros" - - -@pytest.fixture -def write_profiles_yml(request): - def _write_profiles_yml(profiles_dir, dbt_profile_contents): - def cleanup(): - rm_file(Path(profiles_dir) / "profiles.yml") - - request.addfinalizer(cleanup) - write_file(yaml.safe_dump(dbt_profile_contents), profiles_dir, "profiles.yml") - - return _write_profiles_yml - - -# https://gist.github.com/igniteflow/7267431?permalink_comment_id=2551951#gistcomment-2551951 -@contextmanager -def environ(env): - """Temporarily set environment variables inside the context manager and - fully restore previous environment afterwards - """ - original_env = {key: os.getenv(key) for key in env} - os.environ.update(env) - try: - yield - finally: - for key, value in original_env.items(): - if value is None: - del os.environ[key] - else: - os.environ[key] = value - - -class TestProfilesMayNotExist: - def test_debug(self, project): - # The database will not be able to connect; expect neither a pass or a failure (but not an exception) - run_dbt(["debug", "--profiles-dir", "does_not_exist"], expect_pass=None) - - def test_deps(self, project): - run_dbt(["deps", "--profiles-dir", "does_not_exist"]) - - -class TestProfiles: - def dbt_debug(self, project_dir_cli_arg=None, profiles_dir_cli_arg=None): - # begin with no command-line args or user config (from profiles.yml) - # flags.set_from_args(Namespace(), {}) - command = ["debug"] - - if project_dir_cli_arg: - command.extend(["--project-dir", str(project_dir_cli_arg)]) - - if profiles_dir_cli_arg: - command.extend(["--profiles-dir", str(profiles_dir_cli_arg)]) - - # get the output of `dbt debug` regardless of the exit code - return run_dbt_and_capture(command, expect_pass=None) - - @pytest.mark.parametrize( - "project_dir_cli_arg, working_directory", - [ - # 3 different scenarios for `--project-dir` flag and current working directory - (None, "cwd"), # no --project-dir flag and cwd is project directory - (None, "cwd_child"), # no --project-dir flag and cwd is a project subdirectory - ("cwd", "cwd_parent"), # use --project-dir flag and cwd is outside of it - ], - ) - def test_profiles( - self, - project_dir_cli_arg, - working_directory, - write_profiles_yml, - dbt_profile_data, - profiles_home_root, - profiles_project_root, - profiles_flag_root, - profiles_env_root, - request, - ): - """Verify priority order to search for profiles.yml configuration. - - Reverse priority order: - 1. HOME directory - 2. DBT_PROFILES_DIR environment variable - 3. --profiles-dir command-line argument - - Specification later in this list will take priority over earlier ones, even when both are provided. - """ - - # https://pypi.org/project/pytest-lazy-fixture/ is an alternative to using request.getfixturevalue - if project_dir_cli_arg is not None: - project_dir_cli_arg = request.getfixturevalue(project_dir_cli_arg) - - if working_directory is not None: - working_directory = request.getfixturevalue(working_directory) - - # start in the specified directory - if working_directory is not None: - os.chdir(working_directory) - # default case with profiles.yml in the HOME directory - _, stdout = self.dbt_debug(project_dir_cli_arg) - assert f"Using profiles.yml file at {profiles_home_root}" in stdout - - # set DBT_PROFILES_DIR environment variable for the remainder of the cases - env_vars = {"DBT_PROFILES_DIR": profiles_env_root} - with environ(env_vars): - _, stdout = self.dbt_debug(project_dir_cli_arg) - assert f"Using profiles.yml file at {profiles_env_root}" in stdout - - # This additional case is also within the context manager because we want to verify - # that it takes priority even when the relevant environment variable is also set - - # set --profiles-dir on the command-line - _, stdout = self.dbt_debug( - project_dir_cli_arg, profiles_dir_cli_arg=profiles_flag_root - ) - assert f"Using profiles.yml file at {profiles_flag_root}" in stdout diff --git a/tests/functional/profiles/test_profiles_yml.py b/tests/functional/profiles/test_profiles_yml.py deleted file mode 100644 index c4eeabbd..00000000 --- a/tests/functional/profiles/test_profiles_yml.py +++ /dev/null @@ -1,65 +0,0 @@ -from pathlib import Path - -from dbt.cli.main import dbtRunner -from test_profile_dir import environ - - -jinjaesque_password = "no{{jinja{%re{#ndering" - -profile_with_jinjaesque_password = f"""test: - outputs: - default: - dbname: my_db - host: localhost - password: {jinjaesque_password} - port: 12345 - schema: dummy - threads: 4 - type: postgres - user: peter.webb - target: default -""" - -profile_with_env_password = """test: - outputs: - default: - dbname: my_db - host: localhost - password: "{{ env_var('DBT_PASSWORD') }}" - port: 12345 - schema: dummy - threads: 4 - type: postgres - user: peter.webb - target: default -""" - - -class TestProfileParsing: - def write_profiles_yml(self, profiles_root, content) -> None: - with open(Path(profiles_root, "profiles.yml"), "w") as profiles_yml: - profiles_yml.write(content) - - def test_password_not_jinja_rendered_when_invalid(self, project, profiles_root) -> None: - """Verifies that passwords that contain Jinja control characters, but which are - not valid Jinja, do not cause errors.""" - self.write_profiles_yml(profiles_root, profile_with_jinjaesque_password) - - events = [] - result = dbtRunner(callbacks=[events.append]).invoke(["parse"]) - assert result.success - - for e in events: - assert "no{{jinja{%re{#ndering" not in e.info.msg - - def test_password_jinja_rendered_when_valid(self, project, profiles_root) -> None: - """Verifies that a password value that is valid Jinja is rendered as such, - and that it doesn't cause problems if the resulting value looks like Jinja""" - self.write_profiles_yml(profiles_root, profile_with_env_password) - - events = [] - with environ({"DBT_PASSWORD": jinjaesque_password}): - result = dbtRunner(callbacks=[events.append]).invoke(["parse"]) - - assert result.success - assert project.adapter.config.credentials.password == jinjaesque_password diff --git a/tests/functional/schema_tests/test_schema_v2_tests.py b/tests/functional/schema_tests/test_schema_v2_tests.py index a268f096..aae164a1 100644 --- a/tests/functional/schema_tests/test_schema_v2_tests.py +++ b/tests/functional/schema_tests/test_schema_v2_tests.py @@ -2,7 +2,7 @@ import re from dbt.contracts.results import TestStatus -from dbt.exceptions import ParsingError, DuplicateResourceNameError +from dbt.exceptions import ParsingError from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import run_dbt, write_file from dbt_common.exceptions import CompilationError @@ -20,8 +20,6 @@ custom_generic_test_names__schema_yml, custom_generic_test_names_alt_format__model_a, custom_generic_test_names_alt_format__schema_yml, - dupe_generic_tests_collide__model_a, - dupe_generic_tests_collide__schema_yml, ephemeral__ephemeral_sql, ephemeral__schema_yml, invalid_schema_models__model_sql, diff --git a/tests/functional/shared_tests/test_hooks/test_hooks.py b/tests/functional/shared_tests/test_hooks/test_hooks.py index 84381130..7e832038 100644 --- a/tests/functional/shared_tests/test_hooks/test_hooks.py +++ b/tests/functional/shared_tests/test_hooks/test_hooks.py @@ -2,6 +2,7 @@ This file needs to be in its own directory because it uses a `data` directory. Placing this file in its own directory avoids collisions. """ + from dbt.tests.adapter.hooks.test_model_hooks import ( BasePrePostModelHooks, BaseHookRefs, diff --git a/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py b/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py index cd849788..61664ca9 100644 --- a/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py +++ b/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py @@ -2,6 +2,7 @@ This file needs to be in its own directory because it creates a `data` directory at run time. Placing this file in its own directory avoids collisions. """ + from dbt.tests.adapter.simple_seed.test_seed import ( BaseBasicSeedTests, BaseSeedConfigFullRefreshOn, diff --git a/tests/functional/source_overrides/fixtures.py b/tests/functional/source_overrides/fixtures.py deleted file mode 100644 index f7f49235..00000000 --- a/tests/functional/source_overrides/fixtures.py +++ /dev/null @@ -1,387 +0,0 @@ -import pytest - - -dupe_models__schema2_yml = """ -version: 2 -sources: - - name: my_source - overrides: localdep - schema: "{{ target.schema }}" - database: "{{ target.database }}" - freshness: - error_after: {count: 3, period: day} - tables: - - name: my_table - freshness: null - identifier: my_real_seed - # on the override, the "color" column is only unique, it can be null! - columns: - - name: id - data_tests: - - not_null - - unique - - name: color - data_tests: - - unique - - name: my_other_table - freshness: null - identifier: my_real_other_seed - - name: snapshot_freshness - identifier: snapshot_freshness_base - - freshness: - error_after: {count: 1, period: day} - -""" - -dupe_models__schema1_yml = """ -version: 2 -sources: - - name: my_source - overrides: localdep - schema: "{{ target.schema }}" - database: "{{ target.database }}" - freshness: - error_after: {count: 3, period: day} - tables: - - name: my_table - freshness: null - identifier: my_real_seed - # on the override, the "color" column is only unique, it can be null! - columns: - - name: id - data_tests: - - not_null - - unique - - name: color - data_tests: - - unique - - name: my_other_table - freshness: null - identifier: my_real_other_seed - - name: snapshot_freshness - identifier: snapshot_freshness_base - loaded_at_field: updated_at - freshness: - error_after: {count: 1, period: day} - -""" - -local_dependency__dbt_project_yml = """ -config-version: 2 -name: localdep - -version: '1.0' - -profile: 'default' - -seeds: - quote_columns: False - -seed-paths: ['seeds'] - -""" - -local_dependency__models__schema_yml = """ -version: 2 -sources: - - name: my_source - schema: invalid_schema - database: invalid_database - freshness: - error_after: {count: 3, period: hour} - tables: - - name: my_table - freshness: null - identifier: my_seed - columns: - - name: id - data_tests: - - unique - - not_null - - name: color - data_tests: - - unique - - not_null - - name: my_other_table - identifier: my_other_seed - columns: - - name: id - data_tests: - - unique - - not_null - - name: letter - data_tests: - - unique - - not_null - - name: snapshot_freshness - identifier: snapshot_freshness_base - loaded_at_field: updated_at - freshness: - error_after: {count: 1, period: hour} - - name: my_other_source - schema: "{{ target.schema }}" - database: "{{ target.database }}" - freshness: - error_after: {count: 1, period: day} - tables: - - name: never_fresh - loaded_at_field: updated_at - -""" - -local_dependency__models__my_model_sql = """ - -{{ config(materialized="table") }} - -with colors as ( - select id, color from {{ source('my_source', 'my_table') }} -), -letters as ( - select id, letter from {{ source('my_source', 'my_other_table') }} -) -select letter, color from colors join letters using (id) - -""" - -local_dependency__seeds__my_other_seed_csv = """id,letter -1,r -2,g -3,b -""" - -local_dependency__seeds__my_seed_csv = """id,color -1,red -2,green -3,blue -""" - -local_dependency__seeds__keep__never_fresh_csv = """favorite_color,id,first_name,email,ip_address,updated_at -blue,1,Larry,lking0@miitbeian.gov.cn,'69.135.206.194',2008-09-12 19:08:31 -blue,2,Larry,lperkins1@toplist.cz,'64.210.133.162',1978-05-09 04:15:14 -blue,3,Anna,amontgomery2@miitbeian.gov.cn,'168.104.64.114',2011-10-16 04:07:57 -blue,4,Sandra,sgeorge3@livejournal.com,'229.235.252.98',1973-07-19 10:52:43 -blue,5,Fred,fwoods4@google.cn,'78.229.170.124',2012-09-30 16:38:29 -blue,6,Stephen,shanson5@livejournal.com,'182.227.157.105',1995-11-07 21:40:50 -blue,7,William,wmartinez6@upenn.edu,'135.139.249.50',1982-09-05 03:11:59 -blue,8,Jessica,jlong7@hao123.com,'203.62.178.210',1991-10-16 11:03:15 -blue,9,Douglas,dwhite8@tamu.edu,'178.187.247.1',1979-10-01 09:49:48 -blue,10,Lisa,lcoleman9@nydailynews.com,'168.234.128.249',2011-05-26 07:45:49 -blue,11,Ralph,rfieldsa@home.pl,'55.152.163.149',1972-11-18 19:06:11 -blue,12,Louise,lnicholsb@samsung.com,'141.116.153.154',2014-11-25 20:56:14 -blue,13,Clarence,cduncanc@sfgate.com,'81.171.31.133',2011-11-17 07:02:36 -blue,14,Daniel,dfranklind@omniture.com,'8.204.211.37',1980-09-13 00:09:04 -blue,15,Katherine,klanee@auda.org.au,'176.96.134.59',1997-08-22 19:36:56 -blue,16,Billy,bwardf@wikia.com,'214.108.78.85',2003-10-19 02:14:47 -blue,17,Annie,agarzag@ocn.ne.jp,'190.108.42.70',1988-10-28 15:12:35 -blue,18,Shirley,scolemanh@fastcompany.com,'109.251.164.84',1988-08-24 10:50:57 -blue,19,Roger,rfrazieri@scribd.com,'38.145.218.108',1985-12-31 15:17:15 -blue,20,Lillian,lstanleyj@goodreads.com,'47.57.236.17',1970-06-08 02:09:05 -blue,21,Aaron,arodriguezk@nps.gov,'205.245.118.221',1985-10-11 23:07:49 -blue,22,Patrick,pparkerl@techcrunch.com,'19.8.100.182',2006-03-29 12:53:56 -blue,23,Phillip,pmorenom@intel.com,'41.38.254.103',2011-11-07 15:35:43 -blue,24,Henry,hgarcian@newsvine.com,'1.191.216.252',2008-08-28 08:30:44 -blue,25,Irene,iturnero@opera.com,'50.17.60.190',1994-04-01 07:15:02 -blue,26,Andrew,adunnp@pen.io,'123.52.253.176',2000-11-01 06:03:25 -blue,27,David,dgutierrezq@wp.com,'238.23.203.42',1988-01-25 07:29:18 -blue,28,Henry,hsanchezr@cyberchimps.com,'248.102.2.185',1983-01-01 13:36:37 -blue,29,Evelyn,epetersons@gizmodo.com,'32.80.46.119',1979-07-16 17:24:12 -blue,30,Tammy,tmitchellt@purevolume.com,'249.246.167.88',2001-04-03 10:00:23 -blue,31,Jacqueline,jlittleu@domainmarket.com,'127.181.97.47',1986-02-11 21:35:50 -blue,32,Earl,eortizv@opera.com,'166.47.248.240',1996-07-06 08:16:27 -blue,33,Juan,jgordonw@sciencedirect.com,'71.77.2.200',1987-01-31 03:46:44 -blue,34,Diane,dhowellx@nyu.edu,'140.94.133.12',1994-06-11 02:30:05 -blue,35,Randy,rkennedyy@microsoft.com,'73.255.34.196',2005-05-26 20:28:39 -blue,36,Janice,jriveraz@time.com,'22.214.227.32',1990-02-09 04:16:52 -blue,37,Laura,lperry10@diigo.com,'159.148.145.73',2015-03-17 05:59:25 -blue,38,Gary,gray11@statcounter.com,'40.193.124.56',1970-01-27 10:04:51 -blue,39,Jesse,jmcdonald12@typepad.com,'31.7.86.103',2009-03-14 08:14:29 -blue,40,Sandra,sgonzalez13@goodreads.com,'223.80.168.239',1993-05-21 14:08:54 -blue,41,Scott,smoore14@archive.org,'38.238.46.83',1980-08-30 11:16:56 -blue,42,Phillip,pevans15@cisco.com,'158.234.59.34',2011-12-15 23:26:31 -blue,43,Steven,sriley16@google.ca,'90.247.57.68',2011-10-29 19:03:28 -blue,44,Deborah,dbrown17@hexun.com,'179.125.143.240',1995-04-10 14:36:07 -blue,45,Lori,lross18@ow.ly,'64.80.162.180',1980-12-27 16:49:15 -blue,46,Sean,sjackson19@tumblr.com,'240.116.183.69',1988-06-12 21:24:45 -blue,47,Terry,tbarnes1a@163.com,'118.38.213.137',1997-09-22 16:43:19 -blue,48,Dorothy,dross1b@ebay.com,'116.81.76.49',2005-02-28 13:33:24 -blue,49,Samuel,swashington1c@house.gov,'38.191.253.40',1989-01-19 21:15:48 -blue,50,Ralph,rcarter1d@tinyurl.com,'104.84.60.174',2007-08-11 10:21:49 -""" - -local_dependency__seeds__keep__snapshot_freshness_base_csv = """favorite_color,id,first_name,email,ip_address,updated_at -blue,1,Larry,lking0@miitbeian.gov.cn,'69.135.206.194',2008-09-12 19:08:31 -blue,2,Larry,lperkins1@toplist.cz,'64.210.133.162',1978-05-09 04:15:14 -blue,3,Anna,amontgomery2@miitbeian.gov.cn,'168.104.64.114',2011-10-16 04:07:57 -blue,4,Sandra,sgeorge3@livejournal.com,'229.235.252.98',1973-07-19 10:52:43 -blue,5,Fred,fwoods4@google.cn,'78.229.170.124',2012-09-30 16:38:29 -blue,6,Stephen,shanson5@livejournal.com,'182.227.157.105',1995-11-07 21:40:50 -blue,7,William,wmartinez6@upenn.edu,'135.139.249.50',1982-09-05 03:11:59 -blue,8,Jessica,jlong7@hao123.com,'203.62.178.210',1991-10-16 11:03:15 -blue,9,Douglas,dwhite8@tamu.edu,'178.187.247.1',1979-10-01 09:49:48 -blue,10,Lisa,lcoleman9@nydailynews.com,'168.234.128.249',2011-05-26 07:45:49 -blue,11,Ralph,rfieldsa@home.pl,'55.152.163.149',1972-11-18 19:06:11 -blue,12,Louise,lnicholsb@samsung.com,'141.116.153.154',2014-11-25 20:56:14 -blue,13,Clarence,cduncanc@sfgate.com,'81.171.31.133',2011-11-17 07:02:36 -blue,14,Daniel,dfranklind@omniture.com,'8.204.211.37',1980-09-13 00:09:04 -blue,15,Katherine,klanee@auda.org.au,'176.96.134.59',1997-08-22 19:36:56 -blue,16,Billy,bwardf@wikia.com,'214.108.78.85',2003-10-19 02:14:47 -blue,17,Annie,agarzag@ocn.ne.jp,'190.108.42.70',1988-10-28 15:12:35 -blue,18,Shirley,scolemanh@fastcompany.com,'109.251.164.84',1988-08-24 10:50:57 -blue,19,Roger,rfrazieri@scribd.com,'38.145.218.108',1985-12-31 15:17:15 -blue,20,Lillian,lstanleyj@goodreads.com,'47.57.236.17',1970-06-08 02:09:05 -blue,21,Aaron,arodriguezk@nps.gov,'205.245.118.221',1985-10-11 23:07:49 -blue,22,Patrick,pparkerl@techcrunch.com,'19.8.100.182',2006-03-29 12:53:56 -blue,23,Phillip,pmorenom@intel.com,'41.38.254.103',2011-11-07 15:35:43 -blue,24,Henry,hgarcian@newsvine.com,'1.191.216.252',2008-08-28 08:30:44 -blue,25,Irene,iturnero@opera.com,'50.17.60.190',1994-04-01 07:15:02 -blue,26,Andrew,adunnp@pen.io,'123.52.253.176',2000-11-01 06:03:25 -blue,27,David,dgutierrezq@wp.com,'238.23.203.42',1988-01-25 07:29:18 -blue,28,Henry,hsanchezr@cyberchimps.com,'248.102.2.185',1983-01-01 13:36:37 -blue,29,Evelyn,epetersons@gizmodo.com,'32.80.46.119',1979-07-16 17:24:12 -blue,30,Tammy,tmitchellt@purevolume.com,'249.246.167.88',2001-04-03 10:00:23 -blue,31,Jacqueline,jlittleu@domainmarket.com,'127.181.97.47',1986-02-11 21:35:50 -blue,32,Earl,eortizv@opera.com,'166.47.248.240',1996-07-06 08:16:27 -blue,33,Juan,jgordonw@sciencedirect.com,'71.77.2.200',1987-01-31 03:46:44 -blue,34,Diane,dhowellx@nyu.edu,'140.94.133.12',1994-06-11 02:30:05 -blue,35,Randy,rkennedyy@microsoft.com,'73.255.34.196',2005-05-26 20:28:39 -blue,36,Janice,jriveraz@time.com,'22.214.227.32',1990-02-09 04:16:52 -blue,37,Laura,lperry10@diigo.com,'159.148.145.73',2015-03-17 05:59:25 -blue,38,Gary,gray11@statcounter.com,'40.193.124.56',1970-01-27 10:04:51 -blue,39,Jesse,jmcdonald12@typepad.com,'31.7.86.103',2009-03-14 08:14:29 -blue,40,Sandra,sgonzalez13@goodreads.com,'223.80.168.239',1993-05-21 14:08:54 -blue,41,Scott,smoore14@archive.org,'38.238.46.83',1980-08-30 11:16:56 -blue,42,Phillip,pevans15@cisco.com,'158.234.59.34',2011-12-15 23:26:31 -blue,43,Steven,sriley16@google.ca,'90.247.57.68',2011-10-29 19:03:28 -blue,44,Deborah,dbrown17@hexun.com,'179.125.143.240',1995-04-10 14:36:07 -blue,45,Lori,lross18@ow.ly,'64.80.162.180',1980-12-27 16:49:15 -blue,46,Sean,sjackson19@tumblr.com,'240.116.183.69',1988-06-12 21:24:45 -blue,47,Terry,tbarnes1a@163.com,'118.38.213.137',1997-09-22 16:43:19 -blue,48,Dorothy,dross1b@ebay.com,'116.81.76.49',2005-02-28 13:33:24 -blue,49,Samuel,swashington1c@house.gov,'38.191.253.40',1989-01-19 21:15:48 -blue,50,Ralph,rcarter1d@tinyurl.com,'104.84.60.174',2007-08-11 10:21:49 -green,51,Wayne,whudson1e@princeton.edu,'90.61.24.102',1983-07-03 16:58:12 -green,52,Rose,rjames1f@plala.or.jp,'240.83.81.10',1995-06-08 11:46:23 -green,53,Louise,lcox1g@theglobeandmail.com,'105.11.82.145',2016-09-19 14:45:51 -green,54,Kenneth,kjohnson1h@independent.co.uk,'139.5.45.94',1976-08-17 11:26:19 -green,55,Donna,dbrown1i@amazon.co.uk,'19.45.169.45',2006-05-27 16:51:40 -green,56,Johnny,jvasquez1j@trellian.com,'118.202.238.23',1975-11-17 08:42:32 -green,57,Patrick,pramirez1k@tamu.edu,'231.25.153.198',1997-08-06 11:51:09 -green,58,Helen,hlarson1l@prweb.com,'8.40.21.39',1993-08-04 19:53:40 -green,59,Patricia,pspencer1m@gmpg.org,'212.198.40.15',1977-08-03 16:37:27 -green,60,Joseph,jspencer1n@marriott.com,'13.15.63.238',2005-07-23 20:22:06 -green,61,Phillip,pschmidt1o@blogtalkradio.com,'177.98.201.190',1976-05-19 21:47:44 -green,62,Joan,jwebb1p@google.ru,'105.229.170.71',1972-09-07 17:53:47 -green,63,Phyllis,pkennedy1q@imgur.com,'35.145.8.244',2000-01-01 22:33:37 -green,64,Katherine,khunter1r@smh.com.au,'248.168.205.32',1991-01-09 06:40:24 -green,65,Laura,lvasquez1s@wiley.com,'128.129.115.152',1997-10-23 12:04:56 -green,66,Juan,jdunn1t@state.gov,'44.228.124.51',2004-11-10 05:07:35 -green,67,Judith,jholmes1u@wiley.com,'40.227.179.115',1977-08-02 17:01:45 -green,68,Beverly,bbaker1v@wufoo.com,'208.34.84.59',2016-03-06 20:07:23 -green,69,Lawrence,lcarr1w@flickr.com,'59.158.212.223',1988-09-13 06:07:21 -green,70,Gloria,gwilliams1x@mtv.com,'245.231.88.33',1995-03-18 22:32:46 -green,71,Steven,ssims1y@cbslocal.com,'104.50.58.255',2001-08-05 21:26:20 -green,72,Betty,bmills1z@arstechnica.com,'103.177.214.220',1981-12-14 21:26:54 -green,73,Mildred,mfuller20@prnewswire.com,'151.158.8.130',2000-04-19 10:13:55 -green,74,Donald,dday21@icq.com,'9.178.102.255',1972-12-03 00:58:24 -green,75,Eric,ethomas22@addtoany.com,'85.2.241.227',1992-11-01 05:59:30 -green,76,Joyce,jarmstrong23@sitemeter.com,'169.224.20.36',1985-10-24 06:50:01 -green,77,Maria,mmartinez24@amazonaws.com,'143.189.167.135',2005-10-05 05:17:42 -green,78,Harry,hburton25@youtube.com,'156.47.176.237',1978-03-26 05:53:33 -green,79,Kevin,klawrence26@hao123.com,'79.136.183.83',1994-10-12 04:38:52 -green,80,David,dhall27@prweb.com,'133.149.172.153',1976-12-15 16:24:24 -green,81,Kathy,kperry28@twitter.com,'229.242.72.228',1979-03-04 02:58:56 -green,82,Adam,aprice29@elegantthemes.com,'13.145.21.10',1982-11-07 11:46:59 -green,83,Brandon,bgriffin2a@va.gov,'73.249.128.212',2013-10-30 05:30:36 -green,84,Henry,hnguyen2b@discovery.com,'211.36.214.242',1985-01-09 06:37:27 -green,85,Eric,esanchez2c@edublogs.org,'191.166.188.251',2004-05-01 23:21:42 -green,86,Jason,jlee2d@jimdo.com,'193.92.16.182',1973-01-08 09:05:39 -green,87,Diana,drichards2e@istockphoto.com,'19.130.175.245',1994-10-05 22:50:49 -green,88,Andrea,awelch2f@abc.net.au,'94.155.233.96',2002-04-26 08:41:44 -green,89,Louis,lwagner2g@miitbeian.gov.cn,'26.217.34.111',2003-08-25 07:56:39 -green,90,Jane,jsims2h@seesaa.net,'43.4.220.135',1987-03-20 20:39:04 -green,91,Larry,lgrant2i@si.edu,'97.126.79.34',2000-09-07 20:26:19 -green,92,Louis,ldean2j@prnewswire.com,'37.148.40.127',2011-09-16 20:12:14 -green,93,Jennifer,jcampbell2k@xing.com,'38.106.254.142',1988-07-15 05:06:49 -green,94,Wayne,wcunningham2l@google.com.hk,'223.28.26.187',2009-12-15 06:16:54 -green,95,Lori,lstevens2m@icq.com,'181.250.181.58',1984-10-28 03:29:19 -green,96,Judy,jsimpson2n@marriott.com,'180.121.239.219',1986-02-07 15:18:10 -green,97,Phillip,phoward2o@usa.gov,'255.247.0.175',2002-12-26 08:44:45 -green,98,Gloria,gwalker2p@usa.gov,'156.140.7.128',1997-10-04 07:58:58 -green,99,Paul,pjohnson2q@umn.edu,'183.59.198.197',1991-11-14 12:33:55 -green,100,Frank,fgreene2r@blogspot.com,'150.143.68.121',2010-06-12 23:55:39 -""" - -models__schema_yml = """ -version: 2 -sources: - - name: my_source - overrides: localdep - schema: "{{ target.schema }}" - database: "{{ target.database }}" - freshness: - error_after: {count: 3, period: day} - tables: - - name: my_table - freshness: null - identifier: my_real_seed - # on the override, the "color" column is only unique, it can be null! - columns: - - name: id - data_tests: - - not_null - - unique - - name: color - data_tests: - - unique - - name: my_other_table - freshness: null - identifier: my_real_other_seed - - name: snapshot_freshness - identifier: snapshot_freshness_base - loaded_at_field: updated_at - freshness: - error_after: {count: 1, period: day} - -""" - -seeds__expected_result_csv = """letter,color -c,cyan -m,magenta -y,yellow -k,key -""" - -seeds__my_real_other_seed_csv = """id,letter -1,c -2,m -3,y -4,k -""" - -seeds__my_real_seed_csv = """id,color -1,cyan -2,magenta -3,yellow -4,key -5,NULL -""" - - -@pytest.fixture(scope="class") -def local_dependency(): - return { - "dbt_project.yml": local_dependency__dbt_project_yml, - "models": { - "schema.yml": local_dependency__models__schema_yml, - "my_model.sql": local_dependency__models__my_model_sql, - }, - "seeds": { - "my_other_seed.csv": local_dependency__seeds__my_other_seed_csv, - "my_seed.csv": local_dependency__seeds__my_seed_csv, - "keep": { - "never_fresh.csv": local_dependency__seeds__keep__never_fresh_csv, - "snapshot_freshness_base.csv": local_dependency__seeds__keep__snapshot_freshness_base_csv, - }, - }, - } diff --git a/tests/functional/source_overrides/test_simple_source_override.py b/tests/functional/source_overrides/test_simple_source_override.py deleted file mode 100644 index d1cd3352..00000000 --- a/tests/functional/source_overrides/test_simple_source_override.py +++ /dev/null @@ -1,146 +0,0 @@ -from datetime import datetime, timedelta - -from dbt.tests.fixtures.project import write_project_files -from dbt.tests.util import check_relations_equal, run_dbt, update_config_file -import pytest - -from tests.functional.source_overrides.fixtures import ( - local_dependency, - models__schema_yml, - seeds__expected_result_csv, - seeds__my_real_other_seed_csv, - seeds__my_real_seed_csv, -) - - -class TestSourceOverride: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root, local_dependency): # noqa: F811 - write_project_files(project_root, "local_dependency", local_dependency) - - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": models__schema_yml} - - @pytest.fixture(scope="class") - def seeds(self): - return { - "expected_result.csv": seeds__expected_result_csv, - "my_real_other_seed.csv": seeds__my_real_other_seed_csv, - "my_real_seed.csv": seeds__my_real_seed_csv, - } - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - { - "local": "local_dependency", - }, - ] - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "seeds": { - "localdep": { - "enabled": False, - "keep": { - "enabled": True, - }, - }, - "quote_columns": False, - }, - "sources": { - "localdep": { - "my_other_source": { - "enabled": False, - } - } - }, - } - - def _set_updated_at_to(self, insert_id, delta, project): - insert_time = datetime.utcnow() + delta - timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") - # favorite_color,id,first_name,email,ip_address,updated_at - - quoted_columns = ",".join( - project.adapter.quote(c) - for c in ("favorite_color", "id", "first_name", "email", "ip_address", "updated_at") - ) - - kwargs = { - "schema": project.test_schema, - "time": timestr, - "id": insert_id, - "source": project.adapter.quote("snapshot_freshness_base"), - "quoted_columns": quoted_columns, - } - - raw_code = """INSERT INTO {schema}.{source} - ({quoted_columns}) - VALUES ( - 'blue',{id},'Jake','abc@example.com','192.168.1.1','{time}' - )""".format( - **kwargs - ) - - project.run_sql(raw_code) - - return insert_id + 1 - - def test_source_overrides(self, project): - insert_id = 101 - - run_dbt(["deps"]) - - seed_results = run_dbt(["seed"]) - assert len(seed_results) == 5 - - # There should be 7, as we disabled 1 test of the original 8 - test_results = run_dbt(["test"]) - assert len(test_results) == 7 - - results = run_dbt(["run"]) - assert len(results) == 1 - - check_relations_equal(project.adapter, ["expected_result", "my_model"]) - - # set the updated_at field of this seed to last week - insert_id = self._set_updated_at_to(insert_id, timedelta(days=-7), project) - # if snapshot-freshness fails, freshness just didn't happen! - results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) - # we disabled my_other_source, so we only run the one freshness check - # in - assert len(results) == 1 - # If snapshot-freshness passes, that means error_after was - # applied from the source override but not the source table override - insert_id = self._set_updated_at_to(insert_id, timedelta(days=-2), project) - results = run_dbt( - ["source", "snapshot-freshness"], - expect_pass=False, - ) - assert len(results) == 1 - - insert_id = self._set_updated_at_to(insert_id, timedelta(hours=-12), project) - results = run_dbt(["source", "snapshot-freshness"], expect_pass=True) - assert len(results) == 1 - - # update source to be enabled - new_source_config = { - "sources": { - "localdep": { - "my_other_source": { - "enabled": True, - } - } - } - } - update_config_file(new_source_config, project.project_root, "dbt_project.yml") - - # enable my_other_source, snapshot freshness should fail due to the new - # not-fresh source - results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) - assert len(results) == 2 diff --git a/tests/functional/source_overrides/test_source_overrides_duplicate_model.py b/tests/functional/source_overrides/test_source_overrides_duplicate_model.py deleted file mode 100644 index 0a9ab0d8..00000000 --- a/tests/functional/source_overrides/test_source_overrides_duplicate_model.py +++ /dev/null @@ -1,68 +0,0 @@ -import os - -from dbt.tests.fixtures.project import write_project_files -from dbt.tests.util import run_dbt -from dbt_common.exceptions import CompilationError -import pytest - -from tests.functional.source_overrides.fixtures import ( - dupe_models__schema1_yml, - dupe_models__schema2_yml, - local_dependency, -) - - -class TestSourceOverrideDuplicates: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root, local_dependency): # noqa: F811 - write_project_files(project_root, "local_dependency", local_dependency) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema2.yml": dupe_models__schema2_yml, - "schema1.yml": dupe_models__schema1_yml, - } - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - { - "local": "local_dependency", - }, - ] - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "seeds": { - "localdep": { - "enabled": False, - "keep": { - "enabled": True, - }, - }, - "quote_columns": False, - }, - "sources": { - "localdep": { - "my_other_source": { - "enabled": False, - } - } - }, - } - - def test_source_duplicate_overrides(self, project): - run_dbt(["deps"]) - with pytest.raises(CompilationError) as exc: - run_dbt(["compile"]) - - assert "dbt found two schema.yml entries for the same source named" in str(exc.value) - assert "one of these files" in str(exc.value) - schema1_path = os.path.join("models", "schema1.yml") - schema2_path = os.path.join("models", "schema2.yml") - assert schema1_path in str(exc.value) - assert schema2_path in str(exc.value) diff --git a/tests/functional/test_connection_manager.py b/tests/functional/test_connection_manager.py index 778b2fbb..b44181ff 100644 --- a/tests/functional/test_connection_manager.py +++ b/tests/functional/test_connection_manager.py @@ -1,6 +1,7 @@ from unittest import TestCase, mock -from dbt.adapters.contracts.connection import Connection +from dbt.adapters.contracts.connection import Connection, Identifier +from dbt_common.helper_types import Port import psycopg2 from dbt.adapters.postgres import PostgresCredentials, PostgresConnectionManager @@ -20,13 +21,13 @@ def get_connection(self) -> Connection: credentials = PostgresCredentials( host="localhost", user="test-user", - port=1111, + port=Port(1111), password="test-password", database="test-db", schema="test-schema", retries=2, ) - connection = Connection("postgres", None, credentials) + connection = Connection(Identifier("postgres"), None, credentials) return connection def test_open(self): diff --git a/tests/functional/test_dbt_runner.py b/tests/functional/test_dbt_runner.py index c1e05f0f..d3db2d20 100644 --- a/tests/functional/test_dbt_runner.py +++ b/tests/functional/test_dbt_runner.py @@ -13,21 +13,21 @@ def dbt(self) -> dbtRunner: def test_group_invalid_option(self, dbt: dbtRunner) -> None: res = dbt.invoke(["--invalid-option"]) - assert type(res.exception) == DbtUsageException + assert isinstance(res.exception, DbtUsageException) def test_command_invalid_option(self, dbt: dbtRunner) -> None: res = dbt.invoke(["deps", "--invalid-option"]) - assert type(res.exception) == DbtUsageException + assert isinstance(res.exception, DbtUsageException) def test_command_mutually_exclusive_option(self, dbt: dbtRunner) -> None: res = dbt.invoke(["--warn-error", "--warn-error-options", '{"include": "all"}', "deps"]) - assert type(res.exception) == DbtUsageException + assert isinstance(res.exception, DbtUsageException) res = dbt.invoke(["deps", "--warn-error", "--warn-error-options", '{"include": "all"}']) - assert type(res.exception) == DbtUsageException + assert isinstance(res.exception, DbtUsageException) def test_invalid_command(self, dbt: dbtRunner) -> None: res = dbt.invoke(["invalid-command"]) - assert type(res.exception) == DbtUsageException + assert isinstance(res.exception, DbtUsageException) def test_invoke_version(self, dbt: dbtRunner) -> None: dbt.invoke(["--version"]) @@ -57,14 +57,14 @@ def test_invoke_kwargs(self, project, dbt): def test_invoke_kwargs_project_dir(self, project, dbt): res = dbt.invoke(["run"], project_dir="some_random_project_dir") - assert type(res.exception) == DbtProjectError + assert isinstance(res.exception, DbtProjectError) msg = "No dbt_project.yml found at expected path some_random_project_dir" assert msg in res.exception.msg def test_invoke_kwargs_profiles_dir(self, project, dbt): res = dbt.invoke(["run"], profiles_dir="some_random_profiles_dir") - assert type(res.exception) == DbtProjectError + assert isinstance(res.exception, DbtProjectError) msg = "Could not find profile named 'test'" assert msg in res.exception.msg diff --git a/tests/functional/test_init.py b/tests/functional/test_init.py index 1c3133c8..1c8202b7 100644 --- a/tests/functional/test_init.py +++ b/tests/functional/test_init.py @@ -6,7 +6,6 @@ from dbt_common.exceptions import DbtRuntimeError from dbt.tests.util import run_dbt import pytest -import yaml class TestInitProjectWithExistingProfilesYml: diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index 0f46d285..d98c1e5f 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -2,7 +2,7 @@ from unittest import TestCase, mock import pytest -from dbt.context.manifest import generate_query_header_context +from dbt.context.query_header import generate_query_header_context from dbt.context.providers import generate_runtime_macro_context from dbt.contracts.files import FileHash from dbt.contracts.graph.manifest import ManifestStateCheck @@ -167,7 +167,7 @@ def test_quoting_on_rename(self): @pytest.mark.skip( """ We moved from __version__ to __about__ when establishing `hatch` as our build tool. - However, `adapters.factory.register_adapter` assumes __version__ when determining + However, `adapters.factory.register_adapter` assumes __version__ when determining the adapter version. This test causes an import error """ ) @@ -183,7 +183,7 @@ def test_debug_connection_fail_nopass(self): @pytest.mark.skip( """ We moved from __version__ to __about__ when establishing `hatch` as our build tool. - However, `adapters.factory.register_adapter` assumes __version__ when determining + However, `adapters.factory.register_adapter` assumes __version__ when determining the adapter version. This test causes an import error """ ) diff --git a/tests/unit/test_renamed_relations.py b/tests/unit/test_renamed_relations.py index 49900d8e..29bbabf2 100644 --- a/tests/unit/test_renamed_relations.py +++ b/tests/unit/test_renamed_relations.py @@ -9,8 +9,10 @@ def test_renameable_relation(): identifier="my_table", type=RelationType.Table, ) - assert relation.renameable_relations == frozenset({ - RelationType.View, - RelationType.Table, - RelationType.MaterializedView, - }) + assert relation.renameable_relations == frozenset( + { + RelationType.View, + RelationType.Table, + RelationType.MaterializedView, + } + )