From 3789acc5a7b3f71b4e333ac6e235c62ee0c957f5 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Sat, 2 Oct 2021 10:30:09 -0700 Subject: [PATCH] Move redshift, snowflake, bigquery plugins (#3969) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Consolidate dbt-redshift Start moving unit tests fix typo; rm breakpoints rm references to redshift rm all references to redshift except `integration/100_rpc_test` 🙁 cleanup CI fix tests! fix docs test bc of schema version change * Consolidate dbt-snowflake Continue pulling apart Snowflake tests Fixups Rip out even more * Consolidate dbt-bigquery * Rm redshift, snowflake, bigquery from plugins/ * Comment out rs, sf, bq profiles in 100_rpc_test * Add changelog, readme notes Co-authored-by: Kyle Wigley --- .bumpversion.cfg | 12 - .github/dependabot.yml | 15 - .github/scripts/integration-test-matrix.js | 2 +- .github/workflows/integration.yml | 44 - ARCHITECTURE.md | 2 +- CHANGELOG.md | 1 + Makefile | 24 - core/dbt/clients/gcloud.py | 26 - editable-requirements.txt | 3 - plugins/bigquery/README.md | 33 +- .../dbt/adapters/bigquery/__init__.py | 13 - .../dbt/adapters/bigquery/__version__.py | 1 - .../bigquery/dbt/adapters/bigquery/column.py | 127 -- .../dbt/adapters/bigquery/connections.py | 638 -------- .../bigquery/dbt/adapters/bigquery/impl.py | 870 ----------- .../dbt/adapters/bigquery/relation.py | 80 - .../bigquery/dbt/include/bigquery/__init__.py | 2 - .../dbt/include/bigquery/dbt_project.yml | 5 - .../dbt/include/bigquery/macros/adapters.sql | 196 --- .../dbt/include/bigquery/macros/catalog.sql | 209 --- .../dbt/include/bigquery/macros/etc.sql | 15 - .../bigquery/macros/materializations/copy.sql | 32 - .../macros/materializations/incremental.sql | 189 --- .../bigquery/macros/materializations/seed.sql | 19 - .../macros/materializations/snapshot.sql | 15 - .../macros/materializations/table.sql | 84 -- .../bigquery/macros/materializations/view.sql | 25 - .../dbt/include/bigquery/sample_profiles.yml | 26 - plugins/bigquery/setup.py | 73 - plugins/redshift/README.md | 33 +- .../dbt/adapters/redshift/__init__.py | 15 - .../dbt/adapters/redshift/__version__.py | 1 - .../dbt/adapters/redshift/connections.py | 174 --- .../redshift/dbt/adapters/redshift/impl.py | 88 -- .../dbt/adapters/redshift/relation.py | 15 - .../redshift/dbt/include/redshift/__init__.py | 3 - .../dbt/include/redshift/dbt_project.yml | 5 - .../dbt/include/redshift/macros/adapters.sql | 283 ---- .../dbt/include/redshift/macros/catalog.sql | 242 --- .../materializations/snapshot_merge.sql | 4 - .../dbt/include/redshift/macros/relations.sql | 3 - .../dbt/include/redshift/sample_profiles.yml | 25 - plugins/redshift/setup.py | 70 - plugins/snowflake/README.md | 33 +- .../dbt/adapters/snowflake/__init__.py | 13 - .../dbt/adapters/snowflake/__version__.py | 1 - .../dbt/adapters/snowflake/column.py | 31 - .../dbt/adapters/snowflake/connections.py | 375 ----- .../snowflake/dbt/adapters/snowflake/impl.py | 190 --- .../dbt/adapters/snowflake/relation.py | 14 - .../dbt/include/snowflake/__init__.py | 2 - .../dbt/include/snowflake/dbt_project.yml | 5 - .../dbt/include/snowflake/macros/adapters.sql | 251 ---- .../dbt/include/snowflake/macros/catalog.sql | 67 - .../macros/materializations/incremental.sql | 80 - .../macros/materializations/merge.sql | 44 - .../macros/materializations/seed.sql | 37 - .../macros/materializations/table.sql | 34 - .../macros/materializations/view.sql | 13 - .../dbt/include/snowflake/sample_profiles.yml | 29 - plugins/snowflake/setup.py | 70 - requirements.txt | 3 - scripts/build-dist.sh | 2 +- setup.py | 3 - test.env.sample | 22 - .../incremental_update_cols.sql | 17 - .../incremental_overwrite.sql | 3 - .../seeds-merge-cols-initial/seed.csv | 101 -- .../expected_result.csv | 201 --- .../seeds-merge-cols-update/seed.csv | 204 --- .../001_simple_copy_test/test_simple_copy.py | 304 ---- .../test_varchar_widening.py | 16 - .../test_simple_reference.py | 73 - .../invalidate_bigquery.sql | 12 - .../invalidate_snowflake.sql | 12 - .../macros/test_no_overlaps.sql | 5 - .../004_simple_snapshot_test/seed_bq.sql | 81 - .../test-check-col-snapshots-bq/snapshot.sql | 29 - .../test-snapshots-bq/snapshot.sql | 19 - .../test_simple_snapshot.py | 294 +--- .../test_snapshot_check_cols.py | 16 - .../004_simple_snapshot_test/update_bq.sql | 78 - .../005_simple_seed_test/models-bq/schema.yml | 47 - .../005_simple_seed_test/models-rs/schema.yml | 47 - .../models-snowflake/schema.yml | 47 - .../snowflake-data-config/seed_disabled.csv | 21 - .../snowflake-data-config/seed_enabled.csv | 21 - .../test_seed_type_override.py | 89 -- .../005_simple_seed_test/test_simple_seed.py | 4 - .../test_graph_selection.py | 82 - .../models-v2/bq-models/ephemeral_copy.sql | 8 - .../models-v2/bq-models/schema.yml | 30 - .../models-v2/bq-models/table_summary.sql | 9 - .../test_schema_v2_tests.py | 49 - .../009_data_tests_test/test_data_tests.py | 23 - .../dispatch-macros/cast.sql | 3 - .../macros/before-and-after-bq.sql | 30 - .../014_hook_tests/seed-models-bq/schema.yml | 7 - .../014_hook_tests/seed_model_bigquery.sql | 18 - .../014_hook_tests/seed_run_bigquery.sql | 18 - .../014_hook_tests/test_model_hooks_bq.py | 150 -- .../014_hook_tests/test_run_hooks_bq.py | 102 -- .../016_macro_tests/test_macros.py | 15 - .../020_ephemeral_test/test_ephemeral.py | 11 - .../021_concurrency_test/test_concurrency.py | 16 - .../adapter-models/schema.yml | 39 - .../adapter-models/source.sql | 41 - .../test_flattened_get_columns_in_table.sql | 22 - .../test_get_columns_in_table.sql | 10 - .../expiring_table.sql | 1 - .../case-sensitive-models/fUnKyCaSe.sql | 5 - .../case-sensitive-schemas/model.sql | 5 - .../copy_bad_materialization.sql | 2 - .../copy-failing-models/original.sql | 1 - .../copy-models/additional.sql | 1 - .../copy-models/copy_as_incremental.sql | 2 - .../copy-models/copy_as_several_tables.sql | 1 - .../copy-models/copy_as_table.sql | 1 - .../copy-models/original.sql | 1 - .../022_bigquery_test/copy-models/schema.yml | 6 - .../022_bigquery_test/data/data_seed.csv | 5 - .../incremental_overwrite_date_expected.csv | 5 - .../incremental_overwrite_day_expected.csv | 5 - .../incremental_overwrite_range_expected.csv | 5 - .../incremental_overwrite_time_expected.csv | 5 - .../022_bigquery_test/data/merge_expected.csv | 7 - .../dp-models/confirmation.sql | 20 - .../dp-models/confirmation_noconfig.sql | 20 - .../dp-models/events_20180101.sql | 4 - .../dp-models/events_20180102.sql | 4 - .../dp-models/events_20180103.sql | 4 - .../dp-models/partitioned.sql | 16 - .../dp-models/partitioned_noconfig.sql | 7 - .../dp-models/partitioned_simple.sql | 14 - .../022_bigquery_test/dp-models/schema.yml | 33 - .../execution-project-models/model.sql | 1 - .../execution-project-models/schema.yml | 10 - .../incremental_merge_range.sql | 46 - .../incremental_merge_time.sql | 42 - .../incremental_overwrite_date.sql | 41 - .../incremental_overwrite_day.sql | 41 - .../incremental_overwrite_partitions.sql | 42 - .../incremental_overwrite_range.sql | 46 - .../incremental_overwrite_time.sql | 42 - .../location-models/model.sql | 1 - .../macros/partition_metadata.sql | 19 - .../macros/test_creation.sql | 18 - .../macros/test_int_inference.sql | 36 - .../macros/test_project_for_job_id.sql | 7 - .../macros/wrapped_macros.sql | 43 - .../models/clustered_model.sql | 10 - .../022_bigquery_test/models/fUnKyCaSe.sql | 1 - .../models/labeled_model.sql | 8 - .../models/multi_clustered_model.sql | 10 - .../models/partitioned_model.sql | 9 - .../022_bigquery_test/models/schema.yml | 60 - .../models/sql_header_model.sql | 14 - .../models/sql_header_model_incr.sql | 17 - ...sql_header_model_incr_insert_overwrite.sql | 33 - ...der_model_incr_insert_overwrite_static.sql | 32 - .../022_bigquery_test/models/table_model.sql | 8 - .../022_bigquery_test/models/view_model.sql | 14 - .../partition-models/my_model.sql | 15 - .../partition-models/schema.yml | 6 - .../test_bigquery_adapter_functions.py | 77 - .../test_bigquery_adapter_specific.py | 40 - .../test_bigquery_case_sensitive.py | 34 - .../test_bigquery_changing_partitions.py | 183 --- .../test_bigquery_copy_failing_models.py | 36 - .../test_bigquery_copy_models.py | 41 - .../test_bigquery_date_partitioning.py | 46 - .../test_bigquery_execution_project.py | 23 - .../test_bigquery_location_change.py | 31 - .../test_bigquery_query_results.py | 24 - .../test_bigquery_repeated_records.py | 68 - .../test_bigquery_update_columns.py | 85 -- .../test_incremental_strategies.py | 29 - .../test_simple_bigquery_view.py | 71 - .../description_table.sql | 9 - .../description_table.yml | 7 - .../policy_tag_table.sql | 9 - .../policy_tag_table.yml | 8 - .../custom-db-macros/custom_db.sql | 10 - .../db-models/view_1.sql | 3 - .../db-models/view_2.sql | 2 - .../db-models/view_3.sql | 30 - .../test_custom_database.py | 39 - .../test_custom_schema.py | 53 - .../026_aliases_test/macros/cast.sql | 4 - .../026_aliases_test/test_aliases.py | 21 - .../bq_models/clustered.sql | 9 - .../bq_models/multi_clustered.sql | 9 - .../bq_models/nested_table.sql | 15 - .../bq_models/nested_view.sql | 7 - .../bq_models/schema.yml | 44 - .../bq_models_noschema/disabled.sql | 2 - .../bq_models_noschema/model.sql | 2 - .../models/second_model.sql | 7 +- .../rs_models/model.sql | 7 - .../rs_models/schema.yml | 16 - .../test_docs_generate.py | 1324 +---------------- .../models-bq/statement_actual.sql | 23 - .../snowflake-seed/seed.csv | 101 -- .../snowflake-seed/statement_expected.csv | 3 - .../030_statement_test/test_statements.py | 45 - .../032_concurrent_transaction_test/README.md | 33 - .../macros/udfs.sql | 13 - .../models-incremental/model_1.sql | 9 - .../models-incremental/view_model.sql | 3 - .../models-table/model_1.sql | 5 - .../models-table/view_model.sql | 3 - .../models-view/model_1.sql | 5 - .../models-view/view_model.sql | 3 - .../test_concurrent_transaction.py | 136 -- .../034_redshift_test/models/model.sql | 7 - .../034_redshift_test/seed/seed.csv | 2 - .../test_late_binding_view.py | 39 - .../test_changing_relation_type.py | 54 +- .../data/people.csv | 4 - .../models/.gitkeep | 0 .../models/base_table.sql | 9 - .../models/dependent_model.sql | 8 - .../test_view_binding_dependency.py | 136 -- .../038_caching_test/test_caching.py | 8 - .../040_override_database_test/data/seed.csv | 6 - .../models/subfolder/view_3.sql | 1 - .../models/subfolder/view_4.sql | 5 - .../models/view_1.sql | 7 - .../models/view_2.sql | 6 - .../test_override_database.py | 234 --- .../042_sources_test/test_sources.py | 20 +- .../macros-configs/macros.sql | 4 - .../043_custom_aliases_test/macros/macros.sql | 4 - .../049_dbt_debug_test/test_debug.py | 5 - .../models/expected_warehouse.sql | 2 - .../models/invalid_warehouse.sql | 2 - .../models/override_warehouse.sql | 2 - .../expected_warehouse.sql | 2 - .../override_warehouse.sql | 2 - .../050_warehouse_test/test_warehouses.py | 62 - .../051_query_comments_test/models/x.sql | 7 +- .../test_query_comments.py | 12 - .../models-unquoted/model.sql | 7 - .../052_column_quoting/models/model.sql | 4 - .../052_column_quoting/test_column_quotes.py | 59 - .../bigquery-models/renamed_model.sql | 1 - .../bigquery-models/sources.yml | 6 - .../macros/rename_named_relation.sql | 6 - .../054_adapter_methods_test/seed_bq.sql | 32 - .../test_adapter_methods.py | 98 +- .../056_column_type_tests/bq_models/model.sql | 5 - .../bq_models/schema.yml | 10 - .../bq_models_alter_type/altered_schema.yml | 10 - .../bq_models_alter_type/model.sql | 6 - .../056_column_type_tests/rs_models/model.sql | 17 - .../rs_models/schema.yml | 22 - .../056_column_type_tests/sf_models/model.sql | 18 - .../sf_models/schema.yml | 23 - .../test_alter_column_types.py | 15 - .../test_column_types.py | 29 - .../057_oauth_tests/models/model_1.sql | 1 - .../057_oauth_tests/models/model_2.sql | 1 - .../057_oauth_tests/models/model_3.sql | 3 - .../057_oauth_tests/models/model_4.sql | 3 - .../integration/057_oauth_tests/test_oauth.py | 81 - .../test_source_overrides.py | 3 +- .../060_persist_docs_tests/data/seed.csv | 3 - .../models-bigquery-nested/schema.yml | 19 - .../table_model_nested.sql | 8 - .../view_model_nested.sql | 8 - .../models-column-missing/missing_column.sql | 2 - .../models-column-missing/schema.yml | 8 - .../test_persist_docs.py | 224 --- .../062_defer_state_test/test_defer_state.py | 26 +- .../test_relation_name.py | 5 - .../test_column_comments.py | 17 +- .../test_store_test_failures.py | 55 +- .../models/incremental_append_new_columns.sql | 2 +- .../incremental_append_new_columns_target.sql | 2 +- .../models/incremental_sync_all_columns.sql | 2 +- .../incremental_sync_all_columns_target.sql | 2 +- .../test_incremental_schema.py | 74 +- test/integration/100_rpc_test/TODO | 3 + .../test_execute_fetch_and_serialize.py | 14 +- test/integration/base.py | 130 +- test/unit/test_bigquery_adapter.py | 1018 ------------- test/unit/test_config.py | 34 +- test/unit/test_context.py | 148 +- test/unit/test_macro_resolver.py | 21 +- test/unit/test_parse_manifest.py | 2 +- test/unit/test_parser.py | 4 +- test/unit/test_redshift_adapter.py | 371 ----- test/unit/test_snowflake_adapter.py | 583 -------- tox.ini | 17 +- 294 files changed, 128 insertions(+), 14928 deletions(-) delete mode 100644 core/dbt/clients/gcloud.py delete mode 100644 plugins/bigquery/dbt/adapters/bigquery/__init__.py delete mode 100644 plugins/bigquery/dbt/adapters/bigquery/__version__.py delete mode 100644 plugins/bigquery/dbt/adapters/bigquery/column.py delete mode 100644 plugins/bigquery/dbt/adapters/bigquery/connections.py delete mode 100644 plugins/bigquery/dbt/adapters/bigquery/impl.py delete mode 100644 plugins/bigquery/dbt/adapters/bigquery/relation.py delete mode 100644 plugins/bigquery/dbt/include/bigquery/__init__.py delete mode 100644 plugins/bigquery/dbt/include/bigquery/dbt_project.yml delete mode 100644 plugins/bigquery/dbt/include/bigquery/macros/adapters.sql delete mode 100644 plugins/bigquery/dbt/include/bigquery/macros/catalog.sql delete mode 100644 plugins/bigquery/dbt/include/bigquery/macros/etc.sql delete mode 100644 plugins/bigquery/dbt/include/bigquery/macros/materializations/copy.sql delete mode 100644 plugins/bigquery/dbt/include/bigquery/macros/materializations/incremental.sql delete mode 100644 plugins/bigquery/dbt/include/bigquery/macros/materializations/seed.sql delete mode 100644 plugins/bigquery/dbt/include/bigquery/macros/materializations/snapshot.sql delete mode 100644 plugins/bigquery/dbt/include/bigquery/macros/materializations/table.sql delete mode 100644 plugins/bigquery/dbt/include/bigquery/macros/materializations/view.sql delete mode 100644 plugins/bigquery/dbt/include/bigquery/sample_profiles.yml delete mode 100644 plugins/bigquery/setup.py delete mode 100644 plugins/redshift/dbt/adapters/redshift/__init__.py delete mode 100644 plugins/redshift/dbt/adapters/redshift/__version__.py delete mode 100644 plugins/redshift/dbt/adapters/redshift/connections.py delete mode 100644 plugins/redshift/dbt/adapters/redshift/impl.py delete mode 100644 plugins/redshift/dbt/adapters/redshift/relation.py delete mode 100644 plugins/redshift/dbt/include/redshift/__init__.py delete mode 100644 plugins/redshift/dbt/include/redshift/dbt_project.yml delete mode 100644 plugins/redshift/dbt/include/redshift/macros/adapters.sql delete mode 100644 plugins/redshift/dbt/include/redshift/macros/catalog.sql delete mode 100644 plugins/redshift/dbt/include/redshift/macros/materializations/snapshot_merge.sql delete mode 100644 plugins/redshift/dbt/include/redshift/macros/relations.sql delete mode 100644 plugins/redshift/dbt/include/redshift/sample_profiles.yml delete mode 100644 plugins/redshift/setup.py delete mode 100644 plugins/snowflake/dbt/adapters/snowflake/__init__.py delete mode 100644 plugins/snowflake/dbt/adapters/snowflake/__version__.py delete mode 100644 plugins/snowflake/dbt/adapters/snowflake/column.py delete mode 100644 plugins/snowflake/dbt/adapters/snowflake/connections.py delete mode 100644 plugins/snowflake/dbt/adapters/snowflake/impl.py delete mode 100644 plugins/snowflake/dbt/adapters/snowflake/relation.py delete mode 100644 plugins/snowflake/dbt/include/snowflake/__init__.py delete mode 100644 plugins/snowflake/dbt/include/snowflake/dbt_project.yml delete mode 100644 plugins/snowflake/dbt/include/snowflake/macros/adapters.sql delete mode 100644 plugins/snowflake/dbt/include/snowflake/macros/catalog.sql delete mode 100644 plugins/snowflake/dbt/include/snowflake/macros/materializations/incremental.sql delete mode 100644 plugins/snowflake/dbt/include/snowflake/macros/materializations/merge.sql delete mode 100644 plugins/snowflake/dbt/include/snowflake/macros/materializations/seed.sql delete mode 100644 plugins/snowflake/dbt/include/snowflake/macros/materializations/table.sql delete mode 100644 plugins/snowflake/dbt/include/snowflake/macros/materializations/view.sql delete mode 100644 plugins/snowflake/dbt/include/snowflake/sample_profiles.yml delete mode 100644 plugins/snowflake/setup.py delete mode 100644 test.env.sample delete mode 100644 test/integration/001_simple_copy_test/models-merge-update/incremental_update_cols.sql delete mode 100644 test/integration/001_simple_copy_test/seeds-merge-cols-initial/seed.csv delete mode 100644 test/integration/001_simple_copy_test/seeds-merge-cols-update/expected_result.csv delete mode 100644 test/integration/001_simple_copy_test/seeds-merge-cols-update/seed.csv delete mode 100644 test/integration/004_simple_snapshot_test/invalidate_bigquery.sql delete mode 100644 test/integration/004_simple_snapshot_test/invalidate_snowflake.sql delete mode 100644 test/integration/004_simple_snapshot_test/seed_bq.sql delete mode 100644 test/integration/004_simple_snapshot_test/test-check-col-snapshots-bq/snapshot.sql delete mode 100644 test/integration/004_simple_snapshot_test/test-snapshots-bq/snapshot.sql delete mode 100644 test/integration/004_simple_snapshot_test/update_bq.sql delete mode 100644 test/integration/005_simple_seed_test/models-bq/schema.yml delete mode 100644 test/integration/005_simple_seed_test/models-rs/schema.yml delete mode 100644 test/integration/005_simple_seed_test/models-snowflake/schema.yml delete mode 100644 test/integration/005_simple_seed_test/snowflake-data-config/seed_disabled.csv delete mode 100644 test/integration/005_simple_seed_test/snowflake-data-config/seed_enabled.csv delete mode 100644 test/integration/008_schema_tests_test/models-v2/bq-models/ephemeral_copy.sql delete mode 100644 test/integration/008_schema_tests_test/models-v2/bq-models/schema.yml delete mode 100644 test/integration/008_schema_tests_test/models-v2/bq-models/table_summary.sql delete mode 100644 test/integration/014_hook_tests/macros/before-and-after-bq.sql delete mode 100644 test/integration/014_hook_tests/seed-models-bq/schema.yml delete mode 100644 test/integration/014_hook_tests/seed_model_bigquery.sql delete mode 100644 test/integration/014_hook_tests/seed_run_bigquery.sql delete mode 100644 test/integration/014_hook_tests/test_model_hooks_bq.py delete mode 100644 test/integration/014_hook_tests/test_run_hooks_bq.py delete mode 100644 test/integration/022_bigquery_test/adapter-models/schema.yml delete mode 100644 test/integration/022_bigquery_test/adapter-models/source.sql delete mode 100644 test/integration/022_bigquery_test/adapter-models/test_flattened_get_columns_in_table.sql delete mode 100644 test/integration/022_bigquery_test/adapter-models/test_get_columns_in_table.sql delete mode 100644 test/integration/022_bigquery_test/adapter-specific-models/expiring_table.sql delete mode 100644 test/integration/022_bigquery_test/case-sensitive-models/fUnKyCaSe.sql delete mode 100644 test/integration/022_bigquery_test/case-sensitive-schemas/model.sql delete mode 100644 test/integration/022_bigquery_test/copy-failing-models/copy_bad_materialization.sql delete mode 100644 test/integration/022_bigquery_test/copy-failing-models/original.sql delete mode 100644 test/integration/022_bigquery_test/copy-models/additional.sql delete mode 100644 test/integration/022_bigquery_test/copy-models/copy_as_incremental.sql delete mode 100644 test/integration/022_bigquery_test/copy-models/copy_as_several_tables.sql delete mode 100644 test/integration/022_bigquery_test/copy-models/copy_as_table.sql delete mode 100644 test/integration/022_bigquery_test/copy-models/original.sql delete mode 100644 test/integration/022_bigquery_test/copy-models/schema.yml delete mode 100644 test/integration/022_bigquery_test/data/data_seed.csv delete mode 100644 test/integration/022_bigquery_test/data/incremental_overwrite_date_expected.csv delete mode 100644 test/integration/022_bigquery_test/data/incremental_overwrite_day_expected.csv delete mode 100644 test/integration/022_bigquery_test/data/incremental_overwrite_range_expected.csv delete mode 100644 test/integration/022_bigquery_test/data/incremental_overwrite_time_expected.csv delete mode 100644 test/integration/022_bigquery_test/data/merge_expected.csv delete mode 100644 test/integration/022_bigquery_test/dp-models/confirmation.sql delete mode 100644 test/integration/022_bigquery_test/dp-models/confirmation_noconfig.sql delete mode 100644 test/integration/022_bigquery_test/dp-models/events_20180101.sql delete mode 100644 test/integration/022_bigquery_test/dp-models/events_20180102.sql delete mode 100644 test/integration/022_bigquery_test/dp-models/events_20180103.sql delete mode 100644 test/integration/022_bigquery_test/dp-models/partitioned.sql delete mode 100644 test/integration/022_bigquery_test/dp-models/partitioned_noconfig.sql delete mode 100644 test/integration/022_bigquery_test/dp-models/partitioned_simple.sql delete mode 100644 test/integration/022_bigquery_test/dp-models/schema.yml delete mode 100644 test/integration/022_bigquery_test/execution-project-models/model.sql delete mode 100644 test/integration/022_bigquery_test/execution-project-models/schema.yml delete mode 100644 test/integration/022_bigquery_test/incremental-strategy-models/incremental_merge_range.sql delete mode 100644 test/integration/022_bigquery_test/incremental-strategy-models/incremental_merge_time.sql delete mode 100644 test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_date.sql delete mode 100644 test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_day.sql delete mode 100644 test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_partitions.sql delete mode 100644 test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_range.sql delete mode 100644 test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_time.sql delete mode 100644 test/integration/022_bigquery_test/location-models/model.sql delete mode 100644 test/integration/022_bigquery_test/macros/partition_metadata.sql delete mode 100644 test/integration/022_bigquery_test/macros/test_creation.sql delete mode 100644 test/integration/022_bigquery_test/macros/test_int_inference.sql delete mode 100644 test/integration/022_bigquery_test/macros/test_project_for_job_id.sql delete mode 100644 test/integration/022_bigquery_test/macros/wrapped_macros.sql delete mode 100644 test/integration/022_bigquery_test/models/clustered_model.sql delete mode 100644 test/integration/022_bigquery_test/models/fUnKyCaSe.sql delete mode 100644 test/integration/022_bigquery_test/models/labeled_model.sql delete mode 100644 test/integration/022_bigquery_test/models/multi_clustered_model.sql delete mode 100644 test/integration/022_bigquery_test/models/partitioned_model.sql delete mode 100644 test/integration/022_bigquery_test/models/schema.yml delete mode 100644 test/integration/022_bigquery_test/models/sql_header_model.sql delete mode 100644 test/integration/022_bigquery_test/models/sql_header_model_incr.sql delete mode 100644 test/integration/022_bigquery_test/models/sql_header_model_incr_insert_overwrite.sql delete mode 100644 test/integration/022_bigquery_test/models/sql_header_model_incr_insert_overwrite_static.sql delete mode 100644 test/integration/022_bigquery_test/models/table_model.sql delete mode 100644 test/integration/022_bigquery_test/models/view_model.sql delete mode 100644 test/integration/022_bigquery_test/partition-models/my_model.sql delete mode 100644 test/integration/022_bigquery_test/partition-models/schema.yml delete mode 100644 test/integration/022_bigquery_test/test_bigquery_adapter_functions.py delete mode 100644 test/integration/022_bigquery_test/test_bigquery_adapter_specific.py delete mode 100644 test/integration/022_bigquery_test/test_bigquery_case_sensitive.py delete mode 100644 test/integration/022_bigquery_test/test_bigquery_changing_partitions.py delete mode 100644 test/integration/022_bigquery_test/test_bigquery_copy_failing_models.py delete mode 100644 test/integration/022_bigquery_test/test_bigquery_copy_models.py delete mode 100644 test/integration/022_bigquery_test/test_bigquery_date_partitioning.py delete mode 100644 test/integration/022_bigquery_test/test_bigquery_execution_project.py delete mode 100644 test/integration/022_bigquery_test/test_bigquery_location_change.py delete mode 100644 test/integration/022_bigquery_test/test_bigquery_query_results.py delete mode 100644 test/integration/022_bigquery_test/test_bigquery_repeated_records.py delete mode 100644 test/integration/022_bigquery_test/test_bigquery_update_columns.py delete mode 100644 test/integration/022_bigquery_test/test_incremental_strategies.py delete mode 100644 test/integration/022_bigquery_test/test_simple_bigquery_view.py delete mode 100644 test/integration/022_bigquery_test/update-column-description/description_table.sql delete mode 100644 test/integration/022_bigquery_test/update-column-description/description_table.yml delete mode 100644 test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.sql delete mode 100644 test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.yml delete mode 100644 test/integration/024_custom_schema_test/custom-db-macros/custom_db.sql delete mode 100644 test/integration/024_custom_schema_test/db-models/view_1.sql delete mode 100644 test/integration/024_custom_schema_test/db-models/view_2.sql delete mode 100644 test/integration/024_custom_schema_test/db-models/view_3.sql delete mode 100644 test/integration/024_custom_schema_test/test_custom_database.py delete mode 100644 test/integration/029_docs_generate_tests/bq_models/clustered.sql delete mode 100644 test/integration/029_docs_generate_tests/bq_models/multi_clustered.sql delete mode 100644 test/integration/029_docs_generate_tests/bq_models/nested_table.sql delete mode 100644 test/integration/029_docs_generate_tests/bq_models/nested_view.sql delete mode 100644 test/integration/029_docs_generate_tests/bq_models/schema.yml delete mode 100644 test/integration/029_docs_generate_tests/bq_models_noschema/disabled.sql delete mode 100644 test/integration/029_docs_generate_tests/bq_models_noschema/model.sql delete mode 100644 test/integration/029_docs_generate_tests/rs_models/model.sql delete mode 100644 test/integration/029_docs_generate_tests/rs_models/schema.yml delete mode 100644 test/integration/030_statement_test/models-bq/statement_actual.sql delete mode 100644 test/integration/030_statement_test/snowflake-seed/seed.csv delete mode 100644 test/integration/030_statement_test/snowflake-seed/statement_expected.csv delete mode 100644 test/integration/032_concurrent_transaction_test/README.md delete mode 100644 test/integration/032_concurrent_transaction_test/macros/udfs.sql delete mode 100644 test/integration/032_concurrent_transaction_test/models-incremental/model_1.sql delete mode 100644 test/integration/032_concurrent_transaction_test/models-incremental/view_model.sql delete mode 100644 test/integration/032_concurrent_transaction_test/models-table/model_1.sql delete mode 100644 test/integration/032_concurrent_transaction_test/models-table/view_model.sql delete mode 100644 test/integration/032_concurrent_transaction_test/models-view/model_1.sql delete mode 100644 test/integration/032_concurrent_transaction_test/models-view/view_model.sql delete mode 100644 test/integration/032_concurrent_transaction_test/test_concurrent_transaction.py delete mode 100644 test/integration/034_redshift_test/models/model.sql delete mode 100644 test/integration/034_redshift_test/seed/seed.csv delete mode 100644 test/integration/034_redshift_test/test_late_binding_view.py delete mode 100644 test/integration/036_snowflake_view_dependency_test/data/people.csv delete mode 100644 test/integration/036_snowflake_view_dependency_test/models/.gitkeep delete mode 100644 test/integration/036_snowflake_view_dependency_test/models/base_table.sql delete mode 100644 test/integration/036_snowflake_view_dependency_test/models/dependent_model.sql delete mode 100644 test/integration/036_snowflake_view_dependency_test/test_view_binding_dependency.py delete mode 100644 test/integration/040_override_database_test/data/seed.csv delete mode 100644 test/integration/040_override_database_test/models/subfolder/view_3.sql delete mode 100644 test/integration/040_override_database_test/models/subfolder/view_4.sql delete mode 100644 test/integration/040_override_database_test/models/view_1.sql delete mode 100644 test/integration/040_override_database_test/models/view_2.sql delete mode 100644 test/integration/040_override_database_test/test_override_database.py delete mode 100644 test/integration/050_warehouse_test/models/expected_warehouse.sql delete mode 100644 test/integration/050_warehouse_test/models/invalid_warehouse.sql delete mode 100644 test/integration/050_warehouse_test/models/override_warehouse.sql delete mode 100644 test/integration/050_warehouse_test/project-config-models/expected_warehouse.sql delete mode 100644 test/integration/050_warehouse_test/project-config-models/override_warehouse.sql delete mode 100644 test/integration/050_warehouse_test/test_warehouses.py delete mode 100644 test/integration/054_adapter_methods_test/bigquery-models/renamed_model.sql delete mode 100644 test/integration/054_adapter_methods_test/bigquery-models/sources.yml delete mode 100644 test/integration/054_adapter_methods_test/macros/rename_named_relation.sql delete mode 100644 test/integration/054_adapter_methods_test/seed_bq.sql delete mode 100644 test/integration/056_column_type_tests/bq_models/model.sql delete mode 100644 test/integration/056_column_type_tests/bq_models/schema.yml delete mode 100644 test/integration/056_column_type_tests/bq_models_alter_type/altered_schema.yml delete mode 100644 test/integration/056_column_type_tests/bq_models_alter_type/model.sql delete mode 100644 test/integration/056_column_type_tests/rs_models/model.sql delete mode 100644 test/integration/056_column_type_tests/rs_models/schema.yml delete mode 100644 test/integration/056_column_type_tests/sf_models/model.sql delete mode 100644 test/integration/056_column_type_tests/sf_models/schema.yml delete mode 100644 test/integration/057_oauth_tests/models/model_1.sql delete mode 100644 test/integration/057_oauth_tests/models/model_2.sql delete mode 100644 test/integration/057_oauth_tests/models/model_3.sql delete mode 100644 test/integration/057_oauth_tests/models/model_4.sql delete mode 100644 test/integration/057_oauth_tests/test_oauth.py delete mode 100644 test/integration/060_persist_docs_tests/data/seed.csv delete mode 100644 test/integration/060_persist_docs_tests/models-bigquery-nested/schema.yml delete mode 100644 test/integration/060_persist_docs_tests/models-bigquery-nested/table_model_nested.sql delete mode 100644 test/integration/060_persist_docs_tests/models-bigquery-nested/view_model_nested.sql delete mode 100644 test/integration/060_persist_docs_tests/models-column-missing/missing_column.sql delete mode 100644 test/integration/060_persist_docs_tests/models-column-missing/schema.yml create mode 100644 test/integration/100_rpc_test/TODO delete mode 100644 test/unit/test_bigquery_adapter.py delete mode 100644 test/unit/test_redshift_adapter.py delete mode 100644 test/unit/test_snowflake_adapter.py diff --git a/.bumpversion.cfg b/.bumpversion.cfg index a3b40a014fe..a91a0790d68 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -34,17 +34,5 @@ first_value = 1 [bumpversion:file:plugins/postgres/setup.py] -[bumpversion:file:plugins/redshift/setup.py] - -[bumpversion:file:plugins/snowflake/setup.py] - -[bumpversion:file:plugins/bigquery/setup.py] - [bumpversion:file:plugins/postgres/dbt/adapters/postgres/__version__.py] -[bumpversion:file:plugins/redshift/dbt/adapters/redshift/__version__.py] - -[bumpversion:file:plugins/snowflake/dbt/adapters/snowflake/__version__.py] - -[bumpversion:file:plugins/bigquery/dbt/adapters/bigquery/__version__.py] - diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4550b9137c6..e3158a0bae5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,26 +11,11 @@ updates: schedule: interval: "daily" rebase-strategy: "disabled" - - package-ecosystem: "pip" - directory: "/plugins/bigquery" - schedule: - interval: "daily" - rebase-strategy: "disabled" - package-ecosystem: "pip" directory: "/plugins/postgres" schedule: interval: "daily" rebase-strategy: "disabled" - - package-ecosystem: "pip" - directory: "/plugins/redshift" - schedule: - interval: "daily" - rebase-strategy: "disabled" - - package-ecosystem: "pip" - directory: "/plugins/snowflake" - schedule: - interval: "daily" - rebase-strategy: "disabled" # docker dependencies - package-ecosystem: "docker" diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 89053678ff9..f34124bfbd4 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -1,7 +1,7 @@ module.exports = ({ context }) => { const defaultPythonVersion = "3.8"; const supportedPythonVersions = ["3.6", "3.7", "3.8", "3.9"]; - const supportedAdapters = ["snowflake", "postgres", "bigquery", "redshift"]; + const supportedAdapters = ["postgres"]; // if PR, generate matrix based on files changed and PR labels if (context.eventName.includes("pull_request")) { diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2fd387e2638..1655afdd74e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -91,16 +91,6 @@ jobs: - 'core/**' - 'plugins/postgres/**' - 'dev-requirements.txt' - snowflake: - - 'core/**' - - 'plugins/snowflake/**' - bigquery: - - 'core/**' - - 'plugins/bigquery/**' - redshift: - - 'core/**' - - 'plugins/redshift/**' - - 'plugins/postgres/**' - name: Generate integration test matrix id: generate-matrix @@ -191,40 +181,6 @@ jobs: if: matrix.adapter == 'postgres' run: tox - - name: Run tox (redshift) - if: matrix.adapter == 'redshift' - env: - REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} - REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} - REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} - REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} - REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} - run: tox - - - name: Run tox (snowflake) - if: matrix.adapter == 'snowflake' - env: - SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} - SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} - SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} - SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }} - SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: ${{ secrets.SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN }} - SNOWFLAKE_TEST_OAUTH_CLIENT_ID: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_ID }} - SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET }} - SNOWFLAKE_TEST_ALT_DATABASE: ${{ secrets.SNOWFLAKE_TEST_ALT_DATABASE }} - SNOWFLAKE_TEST_ALT_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_ALT_WAREHOUSE }} - SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }} - SNOWFLAKE_TEST_QUOTED_DATABASE: ${{ secrets.SNOWFLAKE_TEST_QUOTED_DATABASE }} - SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }} - run: tox - - - name: Run tox (bigquery) - if: matrix.adapter == 'bigquery' - env: - BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }} - BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }} - run: tox - - uses: actions/upload-artifact@v2 if: always() with: diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6a5c37ede3c..c6fd4355861 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -26,7 +26,7 @@ This is the docs website code. It comes from the dbt-docs repository, and is gen ## Adapters -dbt uses an adapter-plugin pattern to extend support to different databases, warehouses, query engines, etc. The four core adapters that are in the main repository, contained within the [`plugins`](plugins) subdirectory, are: Postgres Redshift, Snowflake and BigQuery. Other warehouses use adapter plugins defined in separate repositories (e.g. [dbt-spark](https://github.com/dbt-labs/dbt-spark), [dbt-presto](https://github.com/dbt-labs/dbt-presto)). +dbt uses an adapter-plugin pattern to extend support to different databases, warehouses, query engines, etc. For testing and development purposes, the dbt-postgres plugin lives alongside the dbt-core codebase, in the [`plugins`](plugins) subdirectory. Like other adapter plugins, it is a self-contained codebase and package that builds on top of dbt-core. Each adapter is a mix of python, Jinja2, and SQL. The adapter code also makes heavy use of Jinja2 to wrap modular chunks of SQL functionality, define default implementations, and allow plugins to override it. diff --git a/CHANGELOG.md b/CHANGELOG.md index eee7a53eaf6..1a38ab631df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - Enact deprecation for object used as dictionaries when they should be dataclasses. Replace deprecation warning with an exception for the dunder methods of `__iter__` and `__len__` for all superclasses of FakeAPIObject. ([#3897](https://github.com/dbt-labs/dbt/issues/3897)) - Enact deprecation for `adapter-macro` and replace deprecation warning with an exception. ([#3901](https://github.com/dbt-labs/dbt/issues/3901)) - Add warning when trying to put a node under the wrong key. ie. A seed under models in a `schema.yml` file. ([#3899](https://github.com/dbt-labs/dbt/issues/3899)) +- Plugins for `redshift`, `snowflake`, and `bigquery` have moved to separate repos: [`dbt-redshift`](https://github.com/dbt-labs/dbt-redshift), [`dbt-snowflake`](https://github.com/dbt-labs/dbt-snowflake), [`dbt-bigquery`](https://github.com/dbt-labs/dbt-bigquery) Contributors: diff --git a/Makefile b/Makefile index 9cfc4c65807..db22e88d3ea 100644 --- a/Makefile +++ b/Makefile @@ -44,30 +44,6 @@ integration-postgres: .env ## Runs postgres integration tests with py38. integration-postgres-fail-fast: .env ## Runs postgres integration tests with py38 in "fail fast" mode. $(DOCKER_CMD) tox -e py38-postgres -- -x -nauto -.PHONY: integration-redshift -integration-redshift: .env ## Runs redshift integration tests with py38. - $(DOCKER_CMD) tox -e py38-redshift -- -nauto - -.PHONY: integration-redshift-fail-fast -integration-redshift-fail-fast: .env ## Runs redshift integration tests with py38 in "fail fast" mode. - $(DOCKER_CMD) tox -e py38-redshift -- -x -nauto - -.PHONY: integration-snowflake -integration-snowflake: .env ## Runs snowflake integration tests with py38. - $(DOCKER_CMD) tox -e py38-snowflake -- -nauto - -.PHONY: integration-snowflake-fail-fast -integration-snowflake-fail-fast: .env ## Runs snowflake integration tests with py38 in "fail fast" mode. - $(DOCKER_CMD) tox -e py38-snowflake -- -x -nauto - -.PHONY: integration-bigquery -integration-bigquery: .env ## Runs bigquery integration tests with py38. - $(DOCKER_CMD) tox -e py38-bigquery -- -nauto - -.PHONY: integration-bigquery-fail-fast -integration-bigquery-fail-fast: .env ## Runs bigquery integration tests with py38 in "fail fast" mode. - $(DOCKER_CMD) tox -e py38-bigquery -- -x -nauto - .PHONY: setup-db setup-db: ## Setup Postgres database with docker-compose for system testing. docker-compose up -d database diff --git a/core/dbt/clients/gcloud.py b/core/dbt/clients/gcloud.py deleted file mode 100644 index 77ed74fdcf7..00000000000 --- a/core/dbt/clients/gcloud.py +++ /dev/null @@ -1,26 +0,0 @@ -from dbt.logger import GLOBAL_LOGGER as logger -import dbt.exceptions -from dbt.clients.system import run_cmd - -NOT_INSTALLED_MSG = """ -dbt requires the gcloud SDK to be installed to authenticate with BigQuery. -Please download and install the SDK, or use a Service Account instead. - -https://cloud.google.com/sdk/ -""" - - -def gcloud_installed(): - try: - run_cmd('.', ['gcloud', '--version']) - return True - except OSError as e: - logger.debug(e) - return False - - -def setup_default_credentials(): - if gcloud_installed(): - run_cmd('.', ["gcloud", "auth", "application-default", "login"]) - else: - raise dbt.exceptions.RuntimeException(NOT_INSTALLED_MSG) diff --git a/editable-requirements.txt b/editable-requirements.txt index 1c185b37d46..f7695be0700 100644 --- a/editable-requirements.txt +++ b/editable-requirements.txt @@ -1,5 +1,2 @@ -e ./core -e ./plugins/postgres --e ./plugins/redshift --e ./plugins/snowflake --e ./plugins/bigquery diff --git a/plugins/bigquery/README.md b/plugins/bigquery/README.md index 7a1b5f50966..7d38a2fd496 100644 --- a/plugins/bigquery/README.md +++ b/plugins/bigquery/README.md @@ -1,32 +1,3 @@ -

- dbt logo -

+### dbt-bigquery -**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications. - -dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis. - -## dbt-bigquery - -The `dbt-bigquery` package contains all of the code required to make dbt operate on a BigQuery database. For -more information on using dbt with BigQuery, consult [the docs](https://docs.getdbt.com/docs/profile-bigquery). - - -## Find out more - -- Check out the [Introduction to dbt](https://docs.getdbt.com/docs/introduction/). -- Read the [dbt Viewpoint](https://docs.getdbt.com/docs/about/viewpoint/). - -## Join thousands of analysts in the dbt community - -- Join the [chat](http://community.getdbt.com/) on Slack. -- Find community posts on [dbt Discourse](https://discourse.getdbt.com). - -## Reporting bugs and contributing code - -- Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/dbt-labs/dbt/issues/new). -- Want to help us build dbt? Check out the [Contributing Getting Started Guide](https://github.com/dbt-labs/dbt/blob/HEAD/CONTRIBUTING.md) - -## Code of Conduct - -Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt Code of Conduct](https://community.getdbt.com/code-of-conduct). +This plugin has moved! https://github.com/dbt-labs/dbt-bigquery \ No newline at end of file diff --git a/plugins/bigquery/dbt/adapters/bigquery/__init__.py b/plugins/bigquery/dbt/adapters/bigquery/__init__.py deleted file mode 100644 index c85e79b389a..00000000000 --- a/plugins/bigquery/dbt/adapters/bigquery/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from dbt.adapters.bigquery.connections import BigQueryConnectionManager # noqa -from dbt.adapters.bigquery.connections import BigQueryCredentials -from dbt.adapters.bigquery.relation import BigQueryRelation # noqa -from dbt.adapters.bigquery.column import BigQueryColumn # noqa -from dbt.adapters.bigquery.impl import BigQueryAdapter, GrantTarget # noqa - -from dbt.adapters.base import AdapterPlugin -from dbt.include import bigquery - -Plugin = AdapterPlugin( - adapter=BigQueryAdapter, - credentials=BigQueryCredentials, - include_path=bigquery.PACKAGE_PATH) diff --git a/plugins/bigquery/dbt/adapters/bigquery/__version__.py b/plugins/bigquery/dbt/adapters/bigquery/__version__.py deleted file mode 100644 index cebffb39326..00000000000 --- a/plugins/bigquery/dbt/adapters/bigquery/__version__.py +++ /dev/null @@ -1 +0,0 @@ -version = '0.21.0rc1' diff --git a/plugins/bigquery/dbt/adapters/bigquery/column.py b/plugins/bigquery/dbt/adapters/bigquery/column.py deleted file mode 100644 index 6c8a70df1c1..00000000000 --- a/plugins/bigquery/dbt/adapters/bigquery/column.py +++ /dev/null @@ -1,127 +0,0 @@ -from dataclasses import dataclass -from typing import Optional, List, TypeVar, Iterable, Type - -from dbt.adapters.base.column import Column - -from google.cloud.bigquery import SchemaField - -Self = TypeVar('Self', bound='BigQueryColumn') - - -@dataclass(init=False) -class BigQueryColumn(Column): - TYPE_LABELS = { - 'STRING': 'STRING', - 'TIMESTAMP': 'TIMESTAMP', - 'FLOAT': 'FLOAT64', - 'INTEGER': 'INT64', - 'RECORD': 'RECORD', - } - fields: List[Self] - mode: str - - def __init__( - self, - column: str, - dtype: str, - fields: Optional[Iterable[SchemaField]] = None, - mode: str = 'NULLABLE', - ) -> None: - super().__init__(column, dtype) - - if fields is None: - fields = [] - - self.fields = self.wrap_subfields(fields) - self.mode = mode - - @classmethod - def wrap_subfields( - cls: Type[Self], fields: Iterable[SchemaField] - ) -> List[Self]: - return [cls.create_from_field(field) for field in fields] - - @classmethod - def create_from_field(cls: Type[Self], field: SchemaField) -> Self: - return cls( - field.name, - cls.translate_type(field.field_type), - field.fields, - field.mode, - ) - - @classmethod - def _flatten_recursive( - cls: Type[Self], col: Self, prefix: Optional[str] = None - ) -> List[Self]: - if prefix is None: - prefix = [] - - if len(col.fields) == 0: - prefixed_name = ".".join(prefix + [col.column]) - new_col = cls(prefixed_name, col.dtype, col.fields, col.mode) - return [new_col] - - new_fields = [] - for field in col.fields: - new_prefix = prefix + [col.column] - new_fields.extend(cls._flatten_recursive(field, new_prefix)) - - return new_fields - - def flatten(self): - return self._flatten_recursive(self) - - @property - def quoted(self): - return '`{}`'.format(self.column) - - def literal(self, value): - return "cast({} as {})".format(value, self.dtype) - - @property - def data_type(self) -> str: - if self.dtype.upper() == 'RECORD': - subcols = [ - "{} {}".format(col.name, col.data_type) for col in self.fields - ] - field_type = 'STRUCT<{}>'.format(", ".join(subcols)) - - else: - field_type = self.dtype - - if self.mode.upper() == 'REPEATED': - return 'ARRAY<{}>'.format(field_type) - - else: - return field_type - - def is_string(self) -> bool: - return self.dtype.lower() == 'string' - - def is_integer(self) -> bool: - return self.dtype.lower() == 'int64' - - def is_numeric(self) -> bool: - return self.dtype.lower() == 'numeric' - - def is_float(self): - return self.dtype.lower() == 'float64' - - def can_expand_to(self: Self, other_column: Self) -> bool: - """returns True if both columns are strings""" - return self.is_string() and other_column.is_string() - - def __repr__(self) -> str: - return "".format(self.name, self.data_type, - self.mode) - - def column_to_bq_schema(self) -> SchemaField: - """Convert a column to a bigquery schema object. - """ - kwargs = {} - if len(self.fields) > 0: - fields = [field.column_to_bq_schema() for field in self.fields] - kwargs = {"fields": fields} - - return SchemaField(self.name, self.dtype, self.mode, **kwargs) diff --git a/plugins/bigquery/dbt/adapters/bigquery/connections.py b/plugins/bigquery/dbt/adapters/bigquery/connections.py deleted file mode 100644 index 9a9fed58c02..00000000000 --- a/plugins/bigquery/dbt/adapters/bigquery/connections.py +++ /dev/null @@ -1,638 +0,0 @@ -import json -import re -from contextlib import contextmanager -from dataclasses import dataclass -from functools import lru_cache -import agate -from requests.exceptions import ConnectionError -from typing import Optional, Any, Dict, Tuple - -import google.auth -import google.auth.exceptions -import google.cloud.bigquery -import google.cloud.exceptions -from google.api_core import retry, client_info -from google.auth import impersonated_credentials -from google.oauth2 import ( - credentials as GoogleCredentials, - service_account as GoogleServiceAccountCredentials -) - -from dbt.utils import format_bytes, format_rows_number -from dbt.clients import agate_helper, gcloud -from dbt.tracking import active_user -from dbt.contracts.connection import ConnectionState, AdapterResponse -from dbt.exceptions import ( - FailedToConnectException, RuntimeException, DatabaseException -) -from dbt.adapters.base import BaseConnectionManager, Credentials -from dbt.logger import GLOBAL_LOGGER as logger -from dbt.version import __version__ as dbt_version - -from dbt.dataclass_schema import StrEnum - - -BQ_QUERY_JOB_SPLIT = '-----Query Job SQL Follows-----' - -WRITE_TRUNCATE = google.cloud.bigquery.job.WriteDisposition.WRITE_TRUNCATE - -REOPENABLE_ERRORS = ( - ConnectionResetError, - ConnectionError, -) - -RETRYABLE_ERRORS = ( - google.cloud.exceptions.ServerError, - google.cloud.exceptions.BadRequest, - ConnectionResetError, - ConnectionError, -) - - -@lru_cache() -def get_bigquery_defaults(scopes=None) -> Tuple[Any, Optional[str]]: - """ - Returns (credentials, project_id) - - project_id is returned available from the environment; otherwise None - """ - # Cached, because the underlying implementation shells out, taking ~1s - return google.auth.default(scopes=scopes) - - -class Priority(StrEnum): - Interactive = 'interactive' - Batch = 'batch' - - -class BigQueryConnectionMethod(StrEnum): - OAUTH = 'oauth' - SERVICE_ACCOUNT = 'service-account' - SERVICE_ACCOUNT_JSON = 'service-account-json' - OAUTH_SECRETS = 'oauth-secrets' - - -@dataclass -class BigQueryAdapterResponse(AdapterResponse): - bytes_processed: Optional[int] = None - - -@dataclass -class BigQueryCredentials(Credentials): - method: BigQueryConnectionMethod - # BigQuery allows an empty database / project, where it defers to the - # environment for the project - database: Optional[str] - execution_project: Optional[str] = None - timeout_seconds: Optional[int] = 300 - location: Optional[str] = None - priority: Optional[Priority] = None - retries: Optional[int] = 1 - maximum_bytes_billed: Optional[int] = None - impersonate_service_account: Optional[str] = None - - # Keyfile json creds - keyfile: Optional[str] = None - keyfile_json: Optional[Dict[str, Any]] = None - - # oauth-secrets - token: Optional[str] = None - refresh_token: Optional[str] = None - client_id: Optional[str] = None - client_secret: Optional[str] = None - token_uri: Optional[str] = None - - _ALIASES = { - 'project': 'database', - 'dataset': 'schema', - 'target_project': 'target_database', - 'target_dataset': 'target_schema', - } - - @property - def type(self): - return 'bigquery' - - @property - def unique_field(self): - return self.database - - def _connection_keys(self): - return ('method', 'database', 'schema', 'location', 'priority', - 'timeout_seconds', 'maximum_bytes_billed') - - @classmethod - def __pre_deserialize__(cls, d: Dict[Any, Any]) -> Dict[Any, Any]: - # We need to inject the correct value of the database (aka project) at - # this stage, ref - # https://github.com/dbt-labs/dbt/pull/2908#discussion_r532927436. - - # `database` is an alias of `project` in BigQuery - if 'database' not in d: - _, database = get_bigquery_defaults() - d['database'] = database - # `execution_project` default to dataset/project - if 'execution_project' not in d: - d['execution_project'] = d['database'] - return d - - -class BigQueryConnectionManager(BaseConnectionManager): - TYPE = 'bigquery' - - SCOPE = ('https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/cloud-platform', - 'https://www.googleapis.com/auth/drive') - - QUERY_TIMEOUT = 300 - RETRIES = 1 - DEFAULT_INITIAL_DELAY = 1.0 # Seconds - DEFAULT_MAXIMUM_DELAY = 1.0 # Seconds - - @classmethod - def handle_error(cls, error, message): - error_msg = "\n".join([item['message'] for item in error.errors]) - raise DatabaseException(error_msg) - - def clear_transaction(self): - pass - - @contextmanager - def exception_handler(self, sql): - try: - yield - - except google.cloud.exceptions.BadRequest as e: - message = "Bad request while running query" - self.handle_error(e, message) - - except google.cloud.exceptions.Forbidden as e: - message = "Access denied while running query" - self.handle_error(e, message) - - except google.auth.exceptions.RefreshError as e: - message = "Unable to generate access token, if you're using " \ - "impersonate_service_account, make sure your " \ - 'initial account has the "roles/' \ - 'iam.serviceAccountTokenCreator" role on the ' \ - 'account you are trying to impersonate.\n\n' \ - f'{str(e)}' - raise RuntimeException(message) - - except Exception as e: - logger.debug("Unhandled error while running:\n{}".format(sql)) - logger.debug(e) - if isinstance(e, RuntimeException): - # during a sql query, an internal to dbt exception was raised. - # this sounds a lot like a signal handler and probably has - # useful information, so raise it without modification. - raise - exc_message = str(e) - # the google bigquery library likes to add the query log, which we - # don't want to log. Hopefully they never change this! - if BQ_QUERY_JOB_SPLIT in exc_message: - exc_message = exc_message.split(BQ_QUERY_JOB_SPLIT)[0].strip() - raise RuntimeException(exc_message) - - def cancel_open(self) -> None: - pass - - @classmethod - def close(cls, connection): - connection.state = ConnectionState.CLOSED - - return connection - - def begin(self): - pass - - def commit(self): - pass - - @classmethod - def get_bigquery_credentials(cls, profile_credentials): - method = profile_credentials.method - creds = GoogleServiceAccountCredentials.Credentials - - if method == BigQueryConnectionMethod.OAUTH: - credentials, _ = get_bigquery_defaults(scopes=cls.SCOPE) - return credentials - - elif method == BigQueryConnectionMethod.SERVICE_ACCOUNT: - keyfile = profile_credentials.keyfile - return creds.from_service_account_file(keyfile, scopes=cls.SCOPE) - - elif method == BigQueryConnectionMethod.SERVICE_ACCOUNT_JSON: - details = profile_credentials.keyfile_json - return creds.from_service_account_info(details, scopes=cls.SCOPE) - - elif method == BigQueryConnectionMethod.OAUTH_SECRETS: - return GoogleCredentials.Credentials( - token=profile_credentials.token, - refresh_token=profile_credentials.refresh_token, - client_id=profile_credentials.client_id, - client_secret=profile_credentials.client_secret, - token_uri=profile_credentials.token_uri, - scopes=cls.SCOPE - ) - - error = ('Invalid `method` in profile: "{}"'.format(method)) - raise FailedToConnectException(error) - - @classmethod - def get_impersonated_bigquery_credentials(cls, profile_credentials): - source_credentials = cls.get_bigquery_credentials(profile_credentials) - return impersonated_credentials.Credentials( - source_credentials=source_credentials, - target_principal=profile_credentials.impersonate_service_account, - target_scopes=list(cls.SCOPE), - lifetime=profile_credentials.timeout_seconds, - ) - - @classmethod - def get_bigquery_client(cls, profile_credentials): - if profile_credentials.impersonate_service_account: - creds =\ - cls.get_impersonated_bigquery_credentials(profile_credentials) - else: - creds = cls.get_bigquery_credentials(profile_credentials) - execution_project = profile_credentials.execution_project - location = getattr(profile_credentials, 'location', None) - - info = client_info.ClientInfo(user_agent=f'dbt-{dbt_version}') - return google.cloud.bigquery.Client( - execution_project, - creds, - location=location, - client_info=info, - ) - - @classmethod - def open(cls, connection): - if connection.state == 'open': - logger.debug('Connection is already open, skipping open.') - return connection - - try: - handle = cls.get_bigquery_client(connection.credentials) - - except google.auth.exceptions.DefaultCredentialsError: - logger.info("Please log into GCP to continue") - gcloud.setup_default_credentials() - - handle = cls.get_bigquery_client(connection.credentials) - - except Exception as e: - logger.debug("Got an error when attempting to create a bigquery " - "client: '{}'".format(e)) - - connection.handle = None - connection.state = 'fail' - - raise FailedToConnectException(str(e)) - - connection.handle = handle - connection.state = 'open' - return connection - - @classmethod - def get_timeout(cls, conn): - credentials = conn.credentials - return credentials.timeout_seconds - - @classmethod - def get_retries(cls, conn) -> int: - credentials = conn.credentials - if credentials.retries is not None: - return credentials.retries - else: - return 1 - - @classmethod - def get_table_from_response(cls, resp): - column_names = [field.name for field in resp.schema] - return agate_helper.table_from_data_flat(resp, column_names) - - def raw_execute(self, sql, fetch=False, *, use_legacy_sql=False): - conn = self.get_thread_connection() - client = conn.handle - - logger.debug('On {}: {}', conn.name, sql) - - if self.profile.query_comment and self.profile.query_comment.job_label: - query_comment = self.query_header.comment.query_comment - labels = self._labels_from_query_comment(query_comment) - else: - labels = {} - - if active_user: - labels['dbt_invocation_id'] = active_user.invocation_id - - job_params = {'use_legacy_sql': use_legacy_sql, 'labels': labels} - - priority = conn.credentials.priority - if priority == Priority.Batch: - job_params['priority'] = google.cloud.bigquery.QueryPriority.BATCH - else: - job_params[ - 'priority'] = google.cloud.bigquery.QueryPriority.INTERACTIVE - - maximum_bytes_billed = conn.credentials.maximum_bytes_billed - if maximum_bytes_billed is not None and maximum_bytes_billed != 0: - job_params['maximum_bytes_billed'] = maximum_bytes_billed - - def fn(): - return self._query_and_results(client, sql, conn, job_params) - - query_job, iterator = self._retry_and_handle(msg=sql, conn=conn, fn=fn) - - return query_job, iterator - - def execute( - self, sql, auto_begin=False, fetch=None - ) -> Tuple[BigQueryAdapterResponse, agate.Table]: - sql = self._add_query_comment(sql) - # auto_begin is ignored on bigquery, and only included for consistency - query_job, iterator = self.raw_execute(sql, fetch=fetch) - - if fetch: - table = self.get_table_from_response(iterator) - else: - table = agate_helper.empty_table() - - message = 'OK' - code = None - num_rows = None - bytes_processed = None - - if query_job.statement_type == 'CREATE_VIEW': - code = 'CREATE VIEW' - - elif query_job.statement_type == 'CREATE_TABLE_AS_SELECT': - conn = self.get_thread_connection() - client = conn.handle - query_table = client.get_table(query_job.destination) - code = 'CREATE TABLE' - num_rows = query_table.num_rows - bytes_processed = query_job.total_bytes_processed - message = '{} ({} rows, {} processed)'.format( - code, - format_rows_number(num_rows), - format_bytes(bytes_processed) - ) - - elif query_job.statement_type == 'SCRIPT': - code = 'SCRIPT' - bytes_processed = query_job.total_bytes_processed - message = f'{code} ({format_bytes(bytes_processed)} processed)' - - elif query_job.statement_type in ['INSERT', 'DELETE', 'MERGE']: - code = query_job.statement_type - num_rows = query_job.num_dml_affected_rows - bytes_processed = query_job.total_bytes_processed - message = '{} ({} rows, {} processed)'.format( - code, - format_rows_number(num_rows), - format_bytes(bytes_processed), - ) - - response = BigQueryAdapterResponse( - _message=message, - rows_affected=num_rows, - code=code, - bytes_processed=bytes_processed - ) - - return response, table - - def get_partitions_metadata(self, table): - def standard_to_legacy(table): - return table.project + ':' + table.dataset + '.' + table.identifier - - legacy_sql = 'SELECT * FROM ['\ - + standard_to_legacy(table) + '$__PARTITIONS_SUMMARY__]' - - sql = self._add_query_comment(legacy_sql) - # auto_begin is ignored on bigquery, and only included for consistency - _, iterator =\ - self.raw_execute(sql, fetch='fetch_result', use_legacy_sql=True) - return self.get_table_from_response(iterator) - - def create_bigquery_table(self, database, schema, table_name, callback, - sql): - """Create a bigquery table. The caller must supply a callback - that takes one argument, a `google.cloud.bigquery.Table`, and mutates - it. - """ - conn = self.get_thread_connection() - client = conn.handle - - view_ref = self.table_ref(database, schema, table_name, conn) - view = google.cloud.bigquery.Table(view_ref) - callback(view) - - def fn(): - return client.create_table(view) - self._retry_and_handle(msg=sql, conn=conn, fn=fn) - - def create_view(self, database, schema, table_name, sql): - def callback(table): - table.view_query = sql - table.view_use_legacy_sql = False - - self.create_bigquery_table(database, schema, table_name, callback, sql) - - def create_table(self, database, schema, table_name, sql): - conn = self.get_thread_connection() - client = conn.handle - - table_ref = self.table_ref(database, schema, table_name, conn) - job_params = {'destination': table_ref, - 'write_disposition': WRITE_TRUNCATE} - - timeout = self.get_timeout(conn) - - def fn(): - return self._query_and_results(client, sql, conn, job_params, - timeout=timeout) - self._retry_and_handle(msg=sql, conn=conn, fn=fn) - - def create_date_partitioned_table(self, database, schema, table_name): - def callback(table): - table.partitioning_type = 'DAY' - - self.create_bigquery_table(database, schema, table_name, callback, - 'CREATE DAY PARTITIONED TABLE') - - def copy_bq_table(self, source, destination, write_disposition): - conn = self.get_thread_connection() - client = conn.handle - -# ------------------------------------------------------------------------------- -# BigQuery allows to use copy API using two different formats: -# 1. client.copy_table(source_table_id, destination_table_id) -# where source_table_id = "your-project.source_dataset.source_table" -# 2. client.copy_table(source_table_ids, destination_table_id) -# where source_table_ids = ["your-project.your_dataset.your_table_name", ...] -# Let's use uniform function call and always pass list there -# ------------------------------------------------------------------------------- - if type(source) is not list: - source = [source] - - source_ref_array = [self.table_ref( - src_table.database, src_table.schema, src_table.table, conn) - for src_table in source] - destination_ref = self.table_ref( - destination.database, destination.schema, destination.table, conn) - - logger.debug( - 'Copying table(s) "{}" to "{}" with disposition: "{}"', - ', '.join(source_ref.path for source_ref in source_ref_array), - destination_ref.path, write_disposition) - - def copy_and_results(): - job_config = google.cloud.bigquery.CopyJobConfig( - write_disposition=write_disposition) - copy_job = client.copy_table( - source_ref_array, destination_ref, job_config=job_config) - iterator = copy_job.result(timeout=self.get_timeout(conn)) - return copy_job, iterator - - self._retry_and_handle( - msg='copy table "{}" to "{}"'.format( - ', '.join(source_ref.path for source_ref in source_ref_array), - destination_ref.path), - conn=conn, fn=copy_and_results) - - @staticmethod - def dataset(database, schema, conn): - dataset_ref = conn.handle.dataset(schema, database) - return google.cloud.bigquery.Dataset(dataset_ref) - - @staticmethod - def dataset_from_id(dataset_id): - return google.cloud.bigquery.Dataset.from_string(dataset_id) - - def table_ref(self, database, schema, table_name, conn): - dataset = self.dataset(database, schema, conn) - return dataset.table(table_name) - - def get_bq_table(self, database, schema, identifier): - """Get a bigquery table for a schema/model.""" - conn = self.get_thread_connection() - table_ref = self.table_ref(database, schema, identifier, conn) - return conn.handle.get_table(table_ref) - - def drop_dataset(self, database, schema): - conn = self.get_thread_connection() - dataset = self.dataset(database, schema, conn) - client = conn.handle - - def fn(): - return client.delete_dataset( - dataset, delete_contents=True, not_found_ok=True) - - self._retry_and_handle( - msg='drop dataset', conn=conn, fn=fn) - - def create_dataset(self, database, schema): - conn = self.get_thread_connection() - client = conn.handle - dataset = self.dataset(database, schema, conn) - - def fn(): - return client.create_dataset(dataset, exists_ok=True) - self._retry_and_handle(msg='create dataset', conn=conn, fn=fn) - - def _query_and_results(self, client, sql, conn, job_params, timeout=None): - """Query the client and wait for results.""" - # Cannot reuse job_config if destination is set and ddl is used - job_config = google.cloud.bigquery.QueryJobConfig(**job_params) - query_job = client.query(sql, job_config=job_config) - iterator = query_job.result(timeout=timeout) - - return query_job, iterator - - def _retry_and_handle(self, msg, conn, fn): - """retry a function call within the context of exception_handler.""" - def reopen_conn_on_error(error): - if isinstance(error, REOPENABLE_ERRORS): - logger.warning('Reopening connection after {!r}', error) - self.close(conn) - self.open(conn) - return - - with self.exception_handler(msg): - return retry.retry_target( - target=fn, - predicate=_ErrorCounter(self.get_retries(conn)).count_error, - sleep_generator=self._retry_generator(), - deadline=None, - on_error=reopen_conn_on_error) - - def _retry_generator(self): - """Generates retry intervals that exponentially back off.""" - return retry.exponential_sleep_generator( - initial=self.DEFAULT_INITIAL_DELAY, - maximum=self.DEFAULT_MAXIMUM_DELAY) - - def _labels_from_query_comment(self, comment: str) -> Dict: - try: - comment_labels = json.loads(comment) - except (TypeError, ValueError): - return {'query_comment': _sanitize_label(comment)} - return { - _sanitize_label(key): _sanitize_label(str(value)) - for key, value in comment_labels.items() - } - - -class _ErrorCounter(object): - """Counts errors seen up to a threshold then raises the next error.""" - - def __init__(self, retries): - self.retries = retries - self.error_count = 0 - - def count_error(self, error): - if self.retries == 0: - return False # Don't log - self.error_count += 1 - if _is_retryable(error) and self.error_count <= self.retries: - logger.debug( - 'Retry attempt {} of {} after error: {}', - self.error_count, self.retries, repr(error)) - return True - else: - return False - - -def _is_retryable(error): - """Return true for errors that are unlikely to occur again if retried.""" - if isinstance(error, RETRYABLE_ERRORS): - return True - elif isinstance(error, google.api_core.exceptions.Forbidden) and any( - e['reason'] == 'rateLimitExceeded' for e in error.errors): - return True - return False - - -_SANITIZE_LABEL_PATTERN = re.compile(r"[^a-z0-9_-]") - -_VALIDATE_LABEL_LENGTH_LIMIT = 63 - - -def _sanitize_label(value: str) -> str: - """Return a legal value for a BigQuery label.""" - value = value.strip().lower() - value = _SANITIZE_LABEL_PATTERN.sub("_", value) - value_length = len(value) - if value_length > _VALIDATE_LABEL_LENGTH_LIMIT: - error_msg = ( - f"Job label length {value_length} is greater than length limit: " - f"{_VALIDATE_LABEL_LENGTH_LIMIT}\n" - f"Current sanitized label: {value}" - ) - raise RuntimeException(error_msg) - else: - return value diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py deleted file mode 100644 index 24aa1cb36a2..00000000000 --- a/plugins/bigquery/dbt/adapters/bigquery/impl.py +++ /dev/null @@ -1,870 +0,0 @@ -from dataclasses import dataclass -from typing import Dict, List, Optional, Any, Set, Union -from dbt.dataclass_schema import dbtClassMixin, ValidationError - -import dbt.deprecations -import dbt.exceptions -import dbt.clients.gcloud -import dbt.clients.agate_helper - -from dbt import ui -from dbt.adapters.base import ( - BaseAdapter, available, RelationType, SchemaSearchMap, AdapterConfig -) -from dbt.adapters.bigquery.relation import BigQueryRelation -from dbt.adapters.bigquery import BigQueryColumn -from dbt.adapters.bigquery import BigQueryConnectionManager -from dbt.contracts.graph.manifest import Manifest -from dbt.logger import GLOBAL_LOGGER as logger, print_timestamped_line -from dbt.utils import filter_null_values - -import google.auth -import google.api_core -import google.oauth2 -import google.cloud.exceptions -import google.cloud.bigquery - -from google.cloud.bigquery import AccessEntry, SchemaField - -import time -import agate -import json - -# Write dispositions for bigquery. -WRITE_APPEND = google.cloud.bigquery.job.WriteDisposition.WRITE_APPEND -WRITE_TRUNCATE = google.cloud.bigquery.job.WriteDisposition.WRITE_TRUNCATE - - -def sql_escape(string): - if not isinstance(string, str): - dbt.exceptions.raise_compiler_exception( - f'cannot escape a non-string: {string}' - ) - - return json.dumps(string)[1:-1] - - -@dataclass -class PartitionConfig(dbtClassMixin): - field: str - data_type: str = 'date' - granularity: str = 'day' - range: Optional[Dict[str, Any]] = None - - def render(self, alias: Optional[str] = None): - column: str = self.field - if alias: - column = f'{alias}.{self.field}' - - if self.data_type.lower() == 'int64' or ( - self.data_type.lower() == 'date' and - self.granularity.lower() == 'day' - ): - return column - else: - return f'{self.data_type}_trunc({column}, {self.granularity})' - - @classmethod - def parse(cls, raw_partition_by) -> Optional['PartitionConfig']: - if raw_partition_by is None: - return None - try: - cls.validate(raw_partition_by) - return cls.from_dict(raw_partition_by) - except ValidationError as exc: - msg = dbt.exceptions.validator_error_message(exc) - dbt.exceptions.raise_compiler_error( - f'Could not parse partition config: {msg}' - ) - except TypeError: - dbt.exceptions.raise_compiler_error( - f'Invalid partition_by config:\n' - f' Got: {raw_partition_by}\n' - f' Expected a dictionary with "field" and "data_type" keys' - ) - - -@dataclass -class GrantTarget(dbtClassMixin): - dataset: str - project: str - - def render(self): - return f'{self.project}.{self.dataset}' - - -def _stub_relation(*args, **kwargs): - return BigQueryRelation.create( - database='', - schema='', - identifier='', - quote_policy={}, - type=BigQueryRelation.Table - ) - - -@dataclass -class BigqueryConfig(AdapterConfig): - cluster_by: Optional[Union[List[str], str]] = None - partition_by: Optional[Dict[str, Any]] = None - kms_key_name: Optional[str] = None - labels: Optional[Dict[str, str]] = None - partitions: Optional[List[str]] = None - grant_access_to: Optional[List[Dict[str, str]]] = None - hours_to_expiration: Optional[int] = None - require_partition_filter: Optional[bool] = None - partition_expiration_days: Optional[int] = None - merge_update_columns: Optional[str] = None - - -class BigQueryAdapter(BaseAdapter): - - RELATION_TYPES = { - 'TABLE': RelationType.Table, - 'VIEW': RelationType.View, - 'EXTERNAL': RelationType.External - } - - Relation = BigQueryRelation - Column = BigQueryColumn - ConnectionManager = BigQueryConnectionManager - - AdapterSpecificConfigs = BigqueryConfig - - ### - # Implementations of abstract methods - ### - - @classmethod - def date_function(cls) -> str: - return 'CURRENT_TIMESTAMP()' - - @classmethod - def is_cancelable(cls) -> bool: - return False - - def drop_relation(self, relation: BigQueryRelation) -> None: - is_cached = self._schema_is_cached(relation.database, relation.schema) - if is_cached: - self.cache_dropped(relation) - - conn = self.connections.get_thread_connection() - client = conn.handle - - dataset = self.connections.dataset(relation.database, relation.schema, - conn) - relation_object = dataset.table(relation.identifier) - client.delete_table(relation_object) - - def truncate_relation(self, relation: BigQueryRelation) -> None: - raise dbt.exceptions.NotImplementedException( - '`truncate` is not implemented for this adapter!' - ) - - def rename_relation( - self, from_relation: BigQueryRelation, to_relation: BigQueryRelation - ) -> None: - - conn = self.connections.get_thread_connection() - client = conn.handle - - from_table_ref = self.connections.table_ref(from_relation.database, - from_relation.schema, - from_relation.identifier, - conn) - from_table = client.get_table(from_table_ref) - if from_table.table_type == "VIEW" or \ - from_relation.type == RelationType.View or \ - to_relation.type == RelationType.View: - raise dbt.exceptions.RuntimeException( - 'Renaming of views is not currently supported in BigQuery' - ) - - to_table_ref = self.connections.table_ref(to_relation.database, - to_relation.schema, - to_relation.identifier, - conn) - - self.cache_renamed(from_relation, to_relation) - client.copy_table(from_table_ref, to_table_ref) - client.delete_table(from_table_ref) - - @available - def list_schemas(self, database: str) -> List[str]: - # the database string we get here is potentially quoted. Strip that off - # for the API call. - database = database.strip('`') - conn = self.connections.get_thread_connection() - client = conn.handle - - def query_schemas(): - # this is similar to how we have to deal with listing tables - all_datasets = client.list_datasets(project=database, - max_results=10000) - return [ds.dataset_id for ds in all_datasets] - - return self.connections._retry_and_handle( - msg='list dataset', conn=conn, fn=query_schemas) - - @available.parse(lambda *a, **k: False) - def check_schema_exists(self, database: str, schema: str) -> bool: - conn = self.connections.get_thread_connection() - client = conn.handle - - bigquery_dataset = self.connections.dataset( - database, schema, conn - ) - # try to do things with the dataset. If it doesn't exist it will 404. - # we have to do it this way to handle underscore-prefixed datasets, - # which appear in neither the information_schema.schemata view nor the - # list_datasets method. - try: - next(iter(client.list_tables(bigquery_dataset, max_results=1))) - except StopIteration: - pass - except google.api_core.exceptions.NotFound: - # the schema does not exist - return False - return True - - def get_columns_in_relation( - self, relation: BigQueryRelation - ) -> List[BigQueryColumn]: - try: - table = self.connections.get_bq_table( - database=relation.database, - schema=relation.schema, - identifier=relation.identifier - ) - return self._get_dbt_columns_from_bq_table(table) - - except (ValueError, google.cloud.exceptions.NotFound) as e: - logger.debug("get_columns_in_relation error: {}".format(e)) - return [] - - def expand_column_types( - self, goal: BigQueryRelation, current: BigQueryRelation - ) -> None: - # This is a no-op on BigQuery - pass - - def expand_target_column_types( - self, from_relation: BigQueryRelation, to_relation: BigQueryRelation - ) -> None: - # This is a no-op on BigQuery - pass - - @available.parse_list - def list_relations_without_caching( - self, schema_relation: BigQueryRelation - ) -> List[BigQueryRelation]: - connection = self.connections.get_thread_connection() - client = connection.handle - - bigquery_dataset = self.connections.dataset( - schema_relation.database, schema_relation.schema, connection - ) - - all_tables = client.list_tables( - bigquery_dataset, - # BigQuery paginates tables by alphabetizing them, and using - # the name of the last table on a page as the key for the - # next page. If that key table gets dropped before we run - # list_relations, then this will 404. So, we avoid this - # situation by making the page size sufficiently large. - # see: https://github.com/dbt-labs/dbt/issues/726 - # TODO: cache the list of relations up front, and then we - # won't need to do this - max_results=100000) - - # This will 404 if the dataset does not exist. This behavior mirrors - # the implementation of list_relations for other adapters - try: - return [self._bq_table_to_relation(table) for table in all_tables] - except google.api_core.exceptions.NotFound: - return [] - - def get_relation( - self, database: str, schema: str, identifier: str - ) -> BigQueryRelation: - if self._schema_is_cached(database, schema): - # if it's in the cache, use the parent's model of going through - # the relations cache and picking out the relation - return super().get_relation( - database=database, - schema=schema, - identifier=identifier - ) - - try: - table = self.connections.get_bq_table(database, schema, identifier) - except google.api_core.exceptions.NotFound: - table = None - return self._bq_table_to_relation(table) - - def create_schema(self, relation: BigQueryRelation) -> None: - database = relation.database - schema = relation.schema - logger.debug('Creating schema "{}.{}".', database, schema) - self.connections.create_dataset(database, schema) - - def drop_schema(self, relation: BigQueryRelation) -> None: - database = relation.database - schema = relation.schema - logger.debug('Dropping schema "{}.{}".', database, schema) - self.connections.drop_dataset(database, schema) - self.cache.drop_schema(database, schema) - - @classmethod - def quote(cls, identifier: str) -> str: - return '`{}`'.format(identifier) - - @classmethod - def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str: - return "string" - - @classmethod - def convert_number_type( - cls, agate_table: agate.Table, col_idx: int - ) -> str: - decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) - return "float64" if decimals else "int64" - - @classmethod - def convert_boolean_type( - cls, agate_table: agate.Table, col_idx: int - ) -> str: - return "bool" - - @classmethod - def convert_datetime_type( - cls, agate_table: agate.Table, col_idx: int - ) -> str: - return "datetime" - - @classmethod - def convert_date_type(cls, agate_table: agate.Table, col_idx: int) -> str: - return "date" - - @classmethod - def convert_time_type(cls, agate_table: agate.Table, col_idx: int) -> str: - return "time" - - ### - # Implementation details - ### - def _make_match_kwargs( - self, database: str, schema: str, identifier: str - ) -> Dict[str, str]: - return filter_null_values({ - 'database': database, - 'identifier': identifier, - 'schema': schema, - }) - - def _get_dbt_columns_from_bq_table(self, table) -> List[BigQueryColumn]: - "Translates BQ SchemaField dicts into dbt BigQueryColumn objects" - - columns = [] - for col in table.schema: - # BigQuery returns type labels that are not valid type specifiers - dtype = self.Column.translate_type(col.field_type) - column = self.Column( - col.name, dtype, col.fields, col.mode) - columns.append(column) - - return columns - - def _agate_to_schema( - self, agate_table: agate.Table, column_override: Dict[str, str] - ) -> List[SchemaField]: - """Convert agate.Table with column names to a list of bigquery schemas. - """ - bq_schema = [] - for idx, col_name in enumerate(agate_table.column_names): - inferred_type = self.convert_agate_type(agate_table, idx) - type_ = column_override.get(col_name, inferred_type) - bq_schema.append(SchemaField(col_name, type_)) - return bq_schema - - def _materialize_as_view(self, model: Dict[str, Any]) -> str: - model_database = model.get('database') - model_schema = model.get('schema') - model_alias = model.get('alias') - model_sql = model.get('compiled_sql') - - logger.debug("Model SQL ({}):\n{}".format(model_alias, model_sql)) - self.connections.create_view( - database=model_database, - schema=model_schema, - table_name=model_alias, - sql=model_sql - ) - return "CREATE VIEW" - - def _materialize_as_table( - self, - model: Dict[str, Any], - model_sql: str, - decorator: Optional[str] = None, - ) -> str: - model_database = model.get('database') - model_schema = model.get('schema') - model_alias = model.get('alias') - - if decorator is None: - table_name = model_alias - else: - table_name = "{}${}".format(model_alias, decorator) - - logger.debug("Model SQL ({}):\n{}".format(table_name, model_sql)) - self.connections.create_table( - database=model_database, - schema=model_schema, - table_name=table_name, - sql=model_sql - ) - - return "CREATE TABLE" - - @available.parse(lambda *a, **k: '') - def copy_table(self, source, destination, materialization): - if materialization == 'incremental': - write_disposition = WRITE_APPEND - elif materialization == 'table': - write_disposition = WRITE_TRUNCATE - else: - dbt.exceptions.raise_compiler_error( - 'Copy table materialization must be "copy" or "table", but ' - f"config.get('copy_materialization', 'table') was " - f'{materialization}') - - self.connections.copy_bq_table( - source, destination, write_disposition) - - return "COPY TABLE with materialization: {}".format(materialization) - - @classmethod - def poll_until_job_completes(cls, job, timeout): - retry_count = timeout - - while retry_count > 0 and job.state != 'DONE': - retry_count -= 1 - time.sleep(1) - job.reload() - - if job.state != 'DONE': - raise dbt.exceptions.RuntimeException("BigQuery Timeout Exceeded") - - elif job.error_result: - message = '\n'.join( - error['message'].strip() for error in job.errors - ) - raise dbt.exceptions.RuntimeException(message) - - def _bq_table_to_relation(self, bq_table): - if bq_table is None: - return None - - return self.Relation.create( - database=bq_table.project, - schema=bq_table.dataset_id, - identifier=bq_table.table_id, - quote_policy={ - 'schema': True, - 'identifier': True - }, - type=self.RELATION_TYPES.get( - bq_table.table_type, RelationType.External - ), - ) - - @classmethod - def warning_on_hooks(hook_type): - msg = "{} is not supported in bigquery and will be ignored" - print_timestamped_line( - msg.format(hook_type), ui.COLOR_FG_YELLOW - ) - - @available - def add_query(self, sql, auto_begin=True, bindings=None, - abridge_sql_log=False): - if self.nice_connection_name() in ['on-run-start', 'on-run-end']: - self.warning_on_hooks(self.nice_connection_name()) - else: - raise dbt.exceptions.NotImplementedException( - '`add_query` is not implemented for this adapter!') - - ### - # Special bigquery adapter methods - ### - @available.parse_none - def make_date_partitioned_table(self, relation): - return self.connections.create_date_partitioned_table( - database=relation.database, - schema=relation.schema, - table_name=relation.identifier - ) - - @available.parse(lambda *a, **k: '') - def execute_model(self, model, materialization, sql_override=None, - decorator=None): - - if sql_override is None: - sql_override = model.get('compiled_sql') - - if materialization == 'view': - res = self._materialize_as_view(model) - elif materialization == 'table': - res = self._materialize_as_table(model, sql_override, decorator) - else: - msg = "Invalid relation type: '{}'".format(materialization) - raise dbt.exceptions.RuntimeException(msg, model) - - return res - - def _partitions_match( - self, table, conf_partition: Optional[PartitionConfig] - ) -> bool: - """ - Check if the actual and configured partitions for a table are a match. - BigQuery tables can be replaced if: - - Both tables are not partitioned, OR - - Both tables are partitioned using the exact same configs - - If there is a mismatch, then the table cannot be replaced directly. - """ - is_partitioned = (table.range_partitioning or table.time_partitioning) - - if not is_partitioned and not conf_partition: - return True - elif conf_partition and table.time_partitioning is not None: - table_field = table.time_partitioning.field.lower() - table_granularity = table.partitioning_type.lower() - return table_field == conf_partition.field.lower() \ - and table_granularity == conf_partition.granularity.lower() - elif conf_partition and table.range_partitioning is not None: - dest_part = table.range_partitioning - conf_part = conf_partition.range or {} - - return dest_part.field == conf_partition.field \ - and dest_part.range_.start == conf_part.get('start') \ - and dest_part.range_.end == conf_part.get('end') \ - and dest_part.range_.interval == conf_part.get('interval') - else: - return False - - def _clusters_match(self, table, conf_cluster) -> bool: - """ - Check if the actual and configured clustering columns for a table - are a match. BigQuery tables can be replaced if clustering columns - match exactly. - """ - if isinstance(conf_cluster, str): - conf_cluster = [conf_cluster] - - return table.clustering_fields == conf_cluster - - @available.parse(lambda *a, **k: True) - def is_replaceable( - self, - relation, - conf_partition: Optional[PartitionConfig], - conf_cluster - ) -> bool: - """ - Check if a given partition and clustering column spec for a table - can replace an existing relation in the database. BigQuery does not - allow tables to be replaced with another table that has a different - partitioning spec. This method returns True if the given config spec is - identical to that of the existing table. - """ - if not relation: - return True - - try: - table = self.connections.get_bq_table( - database=relation.database, - schema=relation.schema, - identifier=relation.identifier - ) - except google.cloud.exceptions.NotFound: - return True - - return all(( - self._partitions_match(table, conf_partition), - self._clusters_match(table, conf_cluster) - )) - - @available - def parse_partition_by( - self, raw_partition_by: Any - ) -> Optional[PartitionConfig]: - """ - dbt v0.16.0 expects `partition_by` to be a dictionary where previously - it was a string. Check the type of `partition_by`, raise error - or warning if string, and attempt to convert to dict. - """ - return PartitionConfig.parse(raw_partition_by) - - def get_table_ref_from_relation(self, conn, relation): - return self.connections.table_ref(relation.database, - relation.schema, - relation.identifier, - conn) - - def _update_column_dict(self, bq_column_dict, dbt_columns, parent=''): - """ - Helper function to recursively traverse the schema of a table in the - update_column_descriptions function below. - - bq_column_dict should be a dict as obtained by the to_api_repr() - function of a SchemaField object. - """ - if parent: - dotted_column_name = '{}.{}'.format(parent, bq_column_dict['name']) - else: - dotted_column_name = bq_column_dict['name'] - - if dotted_column_name in dbt_columns: - column_config = dbt_columns[dotted_column_name] - bq_column_dict['description'] = column_config.get('description') - if column_config.get('policy_tags'): - bq_column_dict['policyTags'] = { - 'names': column_config.get('policy_tags') - } - - new_fields = [] - for child_col_dict in bq_column_dict.get('fields', list()): - new_child_column_dict = self._update_column_dict( - child_col_dict, - dbt_columns, - parent=dotted_column_name - ) - new_fields.append(new_child_column_dict) - - bq_column_dict['fields'] = new_fields - - return bq_column_dict - - @available.parse_none - def update_columns(self, relation, columns): - if len(columns) == 0: - return - - conn = self.connections.get_thread_connection() - table_ref = self.get_table_ref_from_relation(conn, relation) - table = conn.handle.get_table(table_ref) - - new_schema = [] - for bq_column in table.schema: - bq_column_dict = bq_column.to_api_repr() - new_bq_column_dict = self._update_column_dict( - bq_column_dict, - columns - ) - new_schema.append(SchemaField.from_api_repr(new_bq_column_dict)) - - new_table = google.cloud.bigquery.Table(table_ref, schema=new_schema) - conn.handle.update_table(new_table, ['schema']) - - @available.parse_none - def update_table_description( - self, database: str, schema: str, identifier: str, description: str - ): - conn = self.connections.get_thread_connection() - client = conn.handle - - table_ref = self.connections.table_ref( - database, - schema, - identifier, - conn - ) - table = client.get_table(table_ref) - table.description = description - client.update_table(table, ['description']) - - @available.parse_none - def alter_table_add_columns(self, relation, columns): - - logger.debug('Adding columns ({}) to table {}".'.format( - columns, relation)) - - conn = self.connections.get_thread_connection() - client = conn.handle - - table_ref = self.connections.table_ref(relation.database, - relation.schema, - relation.identifier, conn) - table = client.get_table(table_ref) - - new_columns = [col.column_to_bq_schema() for col in columns] - new_schema = table.schema + new_columns - - new_table = google.cloud.bigquery.Table(table_ref, schema=new_schema) - client.update_table(new_table, ['schema']) - - @available.parse_none - def load_dataframe(self, database, schema, table_name, agate_table, - column_override): - bq_schema = self._agate_to_schema(agate_table, column_override) - conn = self.connections.get_thread_connection() - client = conn.handle - - table = self.connections.table_ref(database, schema, table_name, conn) - - load_config = google.cloud.bigquery.LoadJobConfig() - load_config.skip_leading_rows = 1 - load_config.schema = bq_schema - - with open(agate_table.original_abspath, "rb") as f: - job = client.load_table_from_file(f, table, rewind=True, - job_config=load_config) - - timeout = self.connections.get_timeout(conn) - with self.connections.exception_handler("LOAD TABLE"): - self.poll_until_job_completes(job, timeout) - - @classmethod - def _catalog_filter_table( - cls, table: agate.Table, manifest: Manifest - ) -> agate.Table: - table = table.rename(column_names={ - col.name: col.name.replace('__', ':') for col in table.columns - }) - return super()._catalog_filter_table(table, manifest) - - def _get_catalog_schemas(self, manifest: Manifest) -> SchemaSearchMap: - candidates = super()._get_catalog_schemas(manifest) - db_schemas: Dict[str, Set[str]] = {} - result = SchemaSearchMap() - - for candidate, schemas in candidates.items(): - database = candidate.database - if database not in db_schemas: - db_schemas[database] = set(self.list_schemas(database)) - if candidate.schema in db_schemas[database]: - result[candidate] = schemas - else: - logger.debug( - 'Skipping catalog for {}.{} - schema does not exist' - .format(database, candidate.schema) - ) - return result - - @available.parse(lambda *a, **k: {}) - def get_common_options( - self, config: Dict[str, Any], node: Dict[str, Any], temporary: bool = False - ) -> Dict[str, Any]: - opts = {} - - if (config.get('hours_to_expiration') is not None) and (not temporary): - expiration = ( - 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL ' - '{} hour)').format(config.get('hours_to_expiration')) - opts['expiration_timestamp'] = expiration - - if config.persist_relation_docs() and 'description' in node: - description = sql_escape(node['description']) - opts['description'] = '"""{}"""'.format(description) - - if config.get('labels'): - labels = config.get('labels', {}) - opts['labels'] = list(labels.items()) - - return opts - - @available.parse(lambda *a, **k: {}) - def get_table_options( - self, config: Dict[str, Any], node: Dict[str, Any], temporary: bool - ) -> Dict[str, Any]: - opts = self.get_common_options(config, node, temporary) - - if temporary: - expiration = 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)' - opts['expiration_timestamp'] = expiration - - if config.get('kms_key_name') is not None: - opts['kms_key_name'] = "'{}'".format(config.get('kms_key_name')) - - if config.get('require_partition_filter'): - opts['require_partition_filter'] = config.get( - 'require_partition_filter') - - if config.get('partition_expiration_days') is not None: - opts['partition_expiration_days'] = config.get( - 'partition_expiration_days') - - return opts - - @available.parse(lambda *a, **k: {}) - def get_view_options( - self, config: Dict[str, Any], node: Dict[str, Any] - ) -> Dict[str, Any]: - opts = self.get_common_options(config, node) - return opts - - @available.parse_none - def grant_access_to(self, entity, entity_type, role, grant_target_dict): - """ - Given an entity, grants it access to a permissioned dataset. - """ - conn = self.connections.get_thread_connection() - client = conn.handle - - GrantTarget.validate(grant_target_dict) - grant_target = GrantTarget.from_dict(grant_target_dict) - dataset = client.get_dataset( - self.connections.dataset_from_id(grant_target.render()) - ) - - if entity_type == 'view': - entity = self.connections.table_ref( - entity.database, - entity.schema, - entity.identifier, - conn).to_api_repr() - - access_entry = AccessEntry(role, entity_type, entity) - access_entries = dataset.access_entries - - if access_entry in access_entries: - logger.debug(f"Access entry {access_entry} " - f"already exists in dataset") - return - - access_entries.append(AccessEntry(role, entity_type, entity)) - dataset.access_entries = access_entries - client.update_dataset(dataset, ['access_entries']) - - def get_rows_different_sql( - self, - relation_a: BigQueryRelation, - relation_b: BigQueryRelation, - column_names: Optional[List[str]] = None, - except_operator='EXCEPT DISTINCT' - ) -> str: - return super().get_rows_different_sql( - relation_a=relation_a, - relation_b=relation_b, - column_names=column_names, - except_operator=except_operator, - ) - - def timestamp_add_sql( - self, add_to: str, number: int = 1, interval: str = 'hour' - ) -> str: - return f'timestamp_add({add_to}, interval {number} {interval})' - - def string_add_sql( - self, add_to: str, value: str, location='append', - ) -> str: - if location == 'append': - return f"concat({add_to}, '{value}')" - elif location == 'prepend': - return f"concat('{value}', {add_to})" - else: - raise dbt.exceptions.RuntimeException( - f'Got an unexpected location value of "{location}"' - ) diff --git a/plugins/bigquery/dbt/adapters/bigquery/relation.py b/plugins/bigquery/dbt/adapters/bigquery/relation.py deleted file mode 100644 index 08f2c8f067b..00000000000 --- a/plugins/bigquery/dbt/adapters/bigquery/relation.py +++ /dev/null @@ -1,80 +0,0 @@ -from dataclasses import dataclass -from typing import Optional - -from dbt.adapters.base.relation import ( - BaseRelation, ComponentName, InformationSchema -) -from dbt.utils import filter_null_values -from typing import TypeVar - - -Self = TypeVar('Self', bound='BigQueryRelation') - - -@dataclass(frozen=True, eq=False, repr=False) -class BigQueryRelation(BaseRelation): - quote_character: str = '`' - - def matches( - self, - database: Optional[str] = None, - schema: Optional[str] = None, - identifier: Optional[str] = None, - ) -> bool: - search = filter_null_values({ - ComponentName.Database: database, - ComponentName.Schema: schema, - ComponentName.Identifier: identifier - }) - - if not search: - # nothing was passed in - pass - - for k, v in search.items(): - if not self._is_exactish_match(k, v): - return False - - return True - - @property - def project(self): - return self.database - - @property - def dataset(self): - return self.schema - - def information_schema( - self, identifier: Optional[str] = None - ) -> 'BigQueryInformationSchema': - return BigQueryInformationSchema.from_relation(self, identifier) - - -@dataclass(frozen=True, eq=False, repr=False) -class BigQueryInformationSchema(InformationSchema): - quote_character: str = '`' - - @classmethod - def get_include_policy(cls, relation, information_schema_view): - schema = True - if information_schema_view in ('SCHEMATA', 'SCHEMATA_OPTIONS', None): - schema = False - - identifier = True - if information_schema_view == '__TABLES__': - identifier = False - - return relation.include_policy.replace( - schema=schema, - identifier=identifier, - ) - - def replace(self, **kwargs): - if 'information_schema_view' in kwargs: - view = kwargs['information_schema_view'] - # we also need to update the include policy, unless the caller did - # in which case it's their problem - if 'include_policy' not in kwargs: - kwargs['include_policy'] = self.get_include_policy(self, view) - return super().replace(**kwargs) diff --git a/plugins/bigquery/dbt/include/bigquery/__init__.py b/plugins/bigquery/dbt/include/bigquery/__init__.py deleted file mode 100644 index 564a3d1e80a..00000000000 --- a/plugins/bigquery/dbt/include/bigquery/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -import os -PACKAGE_PATH = os.path.dirname(__file__) diff --git a/plugins/bigquery/dbt/include/bigquery/dbt_project.yml b/plugins/bigquery/dbt/include/bigquery/dbt_project.yml deleted file mode 100644 index b4e88b7b0a4..00000000000 --- a/plugins/bigquery/dbt/include/bigquery/dbt_project.yml +++ /dev/null @@ -1,5 +0,0 @@ -config-version: 2 -name: dbt_bigquery -version: 1.0 - -macro-paths: ["macros"] diff --git a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql deleted file mode 100644 index 03f70a1d332..00000000000 --- a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql +++ /dev/null @@ -1,196 +0,0 @@ - -{% macro partition_by(partition_config) -%} - {%- if partition_config is none -%} - {% do return('') %} - {%- elif partition_config.data_type | lower in ('date','timestamp','datetime') -%} - partition by {{ partition_config.render() }} - {%- elif partition_config.data_type | lower in ('int64') -%} - {%- set range = partition_config.range -%} - partition by range_bucket( - {{ partition_config.field }}, - generate_array({{ range.start}}, {{ range.end }}, {{ range.interval }}) - ) - {%- endif -%} -{%- endmacro -%} - -{% macro cluster_by(raw_cluster_by) %} - {%- if raw_cluster_by is not none -%} - cluster by {% if raw_cluster_by is string -%} - {% set raw_cluster_by = [raw_cluster_by] %} - {%- endif -%} - {%- for cluster in raw_cluster_by -%} - {{ cluster }} - {%- if not loop.last -%}, {% endif -%} - {%- endfor -%} - - {% endif %} - -{%- endmacro -%} - -{% macro bigquery_options(opts) %} - {% set options -%} - OPTIONS({% for opt_key, opt_val in opts.items() %} - {{ opt_key }}={{ opt_val }}{{ "," if not loop.last }} - {% endfor %}) - {%- endset %} - {%- do return(options) -%} -{%- endmacro -%} - -{% macro bigquery_table_options(config, node, temporary) %} - {% set opts = adapter.get_table_options(config, node, temporary) %} - {%- do return(bigquery_options(opts)) -%} -{%- endmacro -%} - -{% macro bigquery__create_table_as(temporary, relation, sql) -%} - {%- set raw_partition_by = config.get('partition_by', none) -%} - {%- set raw_cluster_by = config.get('cluster_by', none) -%} - {%- set sql_header = config.get('sql_header', none) -%} - - {%- set partition_config = adapter.parse_partition_by(raw_partition_by) -%} - - {{ sql_header if sql_header is not none }} - - create or replace table {{ relation }} - {{ partition_by(partition_config) }} - {{ cluster_by(raw_cluster_by) }} - {{ bigquery_table_options(config, model, temporary) }} - as ( - {{ sql }} - ); - -{%- endmacro -%} - -{% macro bigquery_view_options(config, node) %} - {% set opts = adapter.get_view_options(config, node) %} - {%- do return(bigquery_options(opts)) -%} -{%- endmacro -%} - -{% macro bigquery__create_view_as(relation, sql) -%} - {%- set sql_header = config.get('sql_header', none) -%} - - {{ sql_header if sql_header is not none }} - - create or replace view {{ relation }} - {{ bigquery_view_options(config, model) }} - as {{ sql }}; - -{% endmacro %} - -{% macro bigquery__create_schema(relation) -%} - {{ adapter.create_schema(relation) }} -{% endmacro %} - -{% macro bigquery__drop_schema(relation) -%} - {{ adapter.drop_schema(relation) }} -{% endmacro %} - -{% macro bigquery__drop_relation(relation) -%} - {% call statement('drop_relation') -%} - drop {{ relation.type }} if exists {{ relation }} - {%- endcall %} -{% endmacro %} - -{% macro bigquery__get_columns_in_relation(relation) -%} - {{ return(adapter.get_columns_in_relation(relation)) }} -{% endmacro %} - - -{% macro bigquery__list_relations_without_caching(schema_relation) -%} - {{ return(adapter.list_relations_without_caching(schema_relation)) }} -{%- endmacro %} - - -{% macro bigquery__current_timestamp() -%} - CURRENT_TIMESTAMP() -{%- endmacro %} - - -{% macro bigquery__snapshot_string_as_time(timestamp) -%} - {%- set result = 'TIMESTAMP("' ~ timestamp ~ '")' -%} - {{ return(result) }} -{%- endmacro %} - - -{% macro bigquery__list_schemas(database) -%} - {{ return(adapter.list_schemas(database)) }} -{% endmacro %} - - -{% macro bigquery__check_schema_exists(information_schema, schema) %} - {{ return(adapter.check_schema_exists(information_schema.database, schema)) }} -{% endmacro %} - -{#-- relation-level macro is not implemented. This is handled in the CTAs statement #} -{% macro bigquery__persist_docs(relation, model, for_relation, for_columns) -%} - {% if for_columns and config.persist_column_docs() and model.columns %} - {% do alter_column_comment(relation, model.columns) %} - {% endif %} -{% endmacro %} - -{% macro bigquery__alter_column_comment(relation, column_dict) -%} - {% do adapter.update_columns(relation, column_dict) %} -{% endmacro %} - -{% macro bigquery__rename_relation(from_relation, to_relation) -%} - {% do adapter.rename_relation(from_relation, to_relation) %} -{% endmacro %} - -{% macro bigquery__alter_relation_add_columns(relation, add_columns) %} - - {% set sql -%} - - alter {{ relation.type }} {{ relation }} - {% for column in add_columns %} - add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} - {% endfor %} - - {%- endset -%} - - {{ return(run_query(sql)) }} - -{% endmacro %} - -{% macro bigquery__alter_relation_drop_columns(relation, drop_columns) %} - - {% set sql -%} - - alter {{ relation.type }} {{ relation }} - - {% for column in drop_columns %} - drop column {{ column.name }}{{ ',' if not loop.last }} - {% endfor %} - - {%- endset -%} - - {{ return(run_query(sql)) }} - -{% endmacro %} - - -{% macro bigquery__alter_column_type(relation, column_name, new_column_type) -%} - {# - Changing a column's data type using a query requires you to scan the entire table. - The query charges can be significant if the table is very large. - - https://cloud.google.com/bigquery/docs/manually-changing-schemas#changing_a_columns_data_type - #} - {% set relation_columns = get_columns_in_relation(relation) %} - - {% set sql %} - select - {%- for col in relation_columns -%} - {% if col.column == column_name %} - CAST({{ col.quoted }} AS {{ new_column_type }}) AS {{ col.quoted }} - {%- else %} - {{ col.quoted }} - {%- endif %} - {%- if not loop.last %},{% endif -%} - {%- endfor %} - from {{ relation }} - {% endset %} - - {% call statement('alter_column_type') %} - {{ create_table_as(False, relation, sql)}} - {%- endcall %} - -{% endmacro %} diff --git a/plugins/bigquery/dbt/include/bigquery/macros/catalog.sql b/plugins/bigquery/dbt/include/bigquery/macros/catalog.sql deleted file mode 100644 index 6822d88a6a8..00000000000 --- a/plugins/bigquery/dbt/include/bigquery/macros/catalog.sql +++ /dev/null @@ -1,209 +0,0 @@ - -{% macro bigquery__get_catalog(information_schema, schemas) -%} - - {%- if (schemas | length) == 0 -%} - {# Hopefully nothing cares about the columns we return when there are no rows #} - {%- set query = "select 1 as id limit 0" -%} - {%- else -%} - - {%- set query -%} - with tables as ( - select - project_id as table_database, - dataset_id as table_schema, - table_id as original_table_name, - - concat(project_id, '.', dataset_id, '.', table_id) as relation_id, - - row_count, - size_bytes as size_bytes, - case - when type = 1 then 'table' - when type = 2 then 'view' - else 'external' - end as table_type, - - REGEXP_CONTAINS(table_id, '^.+[0-9]{8}$') and coalesce(type, 0) = 1 as is_date_shard, - REGEXP_EXTRACT(table_id, '^(.+)[0-9]{8}$') as shard_base_name, - REGEXP_EXTRACT(table_id, '^.+([0-9]{8})$') as shard_name - - from {{ information_schema.replace(information_schema_view='__TABLES__') }} - where ( - {%- for schema in schemas -%} - upper(dataset_id) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - ), - - extracted as ( - - select *, - case - when is_date_shard then shard_base_name - else original_table_name - end as table_name - - from tables - - ), - - unsharded_tables as ( - - select - table_database, - table_schema, - table_name, - coalesce(table_type, 'external') as table_type, - is_date_shard, - - struct( - min(shard_name) as shard_min, - max(shard_name) as shard_max, - count(*) as shard_count - ) as table_shards, - - sum(size_bytes) as size_bytes, - sum(row_count) as row_count, - - max(relation_id) as relation_id - - from extracted - group by 1,2,3,4,5 - - ), - - info_schema_columns as ( - - select - concat(table_catalog, '.', table_schema, '.', table_name) as relation_id, - table_catalog as table_database, - table_schema, - table_name, - - -- use the "real" column name from the paths query below - column_name as base_column_name, - ordinal_position as column_index, - - is_partitioning_column, - clustering_ordinal_position - - from {{ information_schema.replace(information_schema_view='COLUMNS') }} - where ordinal_position is not null - - ), - - info_schema_column_paths as ( - - select - concat(table_catalog, '.', table_schema, '.', table_name) as relation_id, - field_path as column_name, - data_type as column_type, - column_name as base_column_name, - description as column_comment - - from {{ information_schema.replace(information_schema_view='COLUMN_FIELD_PATHS') }} - - ), - - columns as ( - - select * except (base_column_name) - from info_schema_columns - join info_schema_column_paths using (relation_id, base_column_name) - - ), - - column_stats as ( - - select - table_database, - table_schema, - table_name, - max(relation_id) as relation_id, - max(case when is_partitioning_column = 'YES' then 1 else 0 end) = 1 as is_partitioned, - max(case when is_partitioning_column = 'YES' then column_name else null end) as partition_column, - max(case when clustering_ordinal_position is not null then 1 else 0 end) = 1 as is_clustered, - array_to_string( - array_agg( - case - when clustering_ordinal_position is not null then column_name - else null - end ignore nulls - order by clustering_ordinal_position - ), ', ' - ) as clustering_columns - - from columns - group by 1,2,3 - - ) - - select - unsharded_tables.table_database, - unsharded_tables.table_schema, - case - when is_date_shard then concat(unsharded_tables.table_name, '*') - else unsharded_tables.table_name - end as table_name, - unsharded_tables.table_type, - - -- coalesce name and type for External tables - these columns are not - -- present in the COLUMN_FIELD_PATHS resultset - coalesce(columns.column_name, '') as column_name, - -- invent a row number to account for nested fields -- BQ does - -- not treat these nested properties as independent fields - row_number() over ( - partition by relation_id - order by columns.column_index, columns.column_name - ) as column_index, - coalesce(columns.column_type, '') as column_type, - columns.column_comment, - - 'Shard count' as `stats__date_shards__label`, - table_shards.shard_count as `stats__date_shards__value`, - 'The number of date shards in this table' as `stats__date_shards__description`, - is_date_shard as `stats__date_shards__include`, - - 'Shard (min)' as `stats__date_shard_min__label`, - table_shards.shard_min as `stats__date_shard_min__value`, - 'The first date shard in this table' as `stats__date_shard_min__description`, - is_date_shard as `stats__date_shard_min__include`, - - 'Shard (max)' as `stats__date_shard_max__label`, - table_shards.shard_max as `stats__date_shard_max__value`, - 'The last date shard in this table' as `stats__date_shard_max__description`, - is_date_shard as `stats__date_shard_max__include`, - - '# Rows' as `stats__num_rows__label`, - row_count as `stats__num_rows__value`, - 'Approximate count of rows in this table' as `stats__num_rows__description`, - (unsharded_tables.table_type = 'table') as `stats__num_rows__include`, - - 'Approximate Size' as `stats__num_bytes__label`, - size_bytes as `stats__num_bytes__value`, - 'Approximate size of table as reported by BigQuery' as `stats__num_bytes__description`, - (unsharded_tables.table_type = 'table') as `stats__num_bytes__include`, - - 'Partitioned By' as `stats__partitioning_type__label`, - partition_column as `stats__partitioning_type__value`, - 'The partitioning column for this table' as `stats__partitioning_type__description`, - is_partitioned as `stats__partitioning_type__include`, - - 'Clustered By' as `stats__clustering_fields__label`, - clustering_columns as `stats__clustering_fields__value`, - 'The clustering columns for this table' as `stats__clustering_fields__description`, - is_clustered as `stats__clustering_fields__include` - - -- join using relation_id (an actual relation, not a shard prefix) to make - -- sure that column metadata is picked up through the join. This will only - -- return the column information for the "max" table in a date-sharded table set - from unsharded_tables - left join columns using (relation_id) - left join column_stats using (relation_id) - {%- endset -%} - - {%- endif -%} - - {{ return(run_query(query)) }} - -{%- endmacro %} diff --git a/plugins/bigquery/dbt/include/bigquery/macros/etc.sql b/plugins/bigquery/dbt/include/bigquery/macros/etc.sql deleted file mode 100644 index 59b61473ed7..00000000000 --- a/plugins/bigquery/dbt/include/bigquery/macros/etc.sql +++ /dev/null @@ -1,15 +0,0 @@ -{% macro date_sharded_table(base_name) %} - {{ return(base_name ~ "[DBT__PARTITION_DATE]") }} -{% endmacro %} - -{% macro grant_access_to(entity, entity_type, role, grant_target_dict) -%} - {% do adapter.grant_access_to(entity, entity_type, role, grant_target_dict) %} -{% endmacro %} - -{%- macro get_partitions_metadata(table) -%} - {%- if execute -%} - {%- set res = adapter.get_partitions_metadata(table) -%} - {{- return(res) -}} - {%- endif -%} - {{- return(None) -}} -{%- endmacro -%} diff --git a/plugins/bigquery/dbt/include/bigquery/macros/materializations/copy.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/copy.sql deleted file mode 100644 index 6a86fbe4418..00000000000 --- a/plugins/bigquery/dbt/include/bigquery/macros/materializations/copy.sql +++ /dev/null @@ -1,32 +0,0 @@ -{% materialization copy, adapter='bigquery' -%} - - {# Setup #} - {{ run_hooks(pre_hooks) }} - - {% set destination = this.incorporate(type='table') %} - - {# there can be several ref() or source() according to BQ copy API docs #} - {# cycle over ref() and source() to create source tables array #} - {% set source_array = [] %} - {% for ref_table in model.refs %} - {{ source_array.append(ref(*ref_table)) }} - {% endfor %} - - {% for src_table in model.sources %} - {{ source_array.append(source(*src_table)) }} - {% endfor %} - - {# Call adapter's copy_table function #} - {%- set result_str = adapter.copy_table( - source_array, - destination, - config.get('copy_materialization', default = 'table')) -%} - - {{ store_result('main', response=result_str) }} - - {# Clean up #} - {{ run_hooks(post_hooks) }} - {{ adapter.commit() }} - - {{ return({'relations': [destination]}) }} -{%- endmaterialization %} diff --git a/plugins/bigquery/dbt/include/bigquery/macros/materializations/incremental.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/incremental.sql deleted file mode 100644 index 0b8e7bac5dd..00000000000 --- a/plugins/bigquery/dbt/include/bigquery/macros/materializations/incremental.sql +++ /dev/null @@ -1,189 +0,0 @@ - -{% macro dbt_bigquery_validate_get_incremental_strategy(config) %} - {#-- Find and validate the incremental strategy #} - {%- set strategy = config.get("incremental_strategy", default="merge") -%} - - {% set invalid_strategy_msg -%} - Invalid incremental strategy provided: {{ strategy }} - Expected one of: 'merge', 'insert_overwrite' - {%- endset %} - {% if strategy not in ['merge', 'insert_overwrite'] %} - {% do exceptions.raise_compiler_error(invalid_strategy_msg) %} - {% endif %} - - {% do return(strategy) %} -{% endmacro %} - - -{% macro bq_insert_overwrite( - tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists -) %} - - {% if partitions is not none and partitions != [] %} {# static #} - - {% set predicate -%} - {{ partition_by.render(alias='DBT_INTERNAL_DEST') }} in ( - {{ partitions | join (', ') }} - ) - {%- endset %} - - {%- set source_sql -%} - ( - {{sql}} - ) - {%- endset -%} - - {{ get_insert_overwrite_merge_sql(target_relation, source_sql, dest_columns, [predicate], include_sql_header=true) }} - - {% else %} {# dynamic #} - - {% set predicate -%} - {{ partition_by.render(alias='DBT_INTERNAL_DEST') }} in unnest(dbt_partitions_for_replacement) - {%- endset %} - - {%- set source_sql -%} - ( - select * from {{ tmp_relation }} - ) - {%- endset -%} - - -- generated script to merge partitions into {{ target_relation }} - declare dbt_partitions_for_replacement array<{{ partition_by.data_type }}>; - declare _dbt_max_partition {{ partition_by.data_type }} default ( - select max({{ partition_by.field }}) from {{ this }} - where {{ partition_by.field }} is not null - ); - - {# have we already created the temp table to check for schema changes? #} - {% if not tmp_relation_exists %} - -- 1. create a temp table - {{ create_table_as(True, tmp_relation, sql) }} - {% else %} - -- 1. temp table already exists, we used it to check for schema changes - {% endif %} - - -- 2. define partitions to update - set (dbt_partitions_for_replacement) = ( - select as struct - array_agg(distinct {{ partition_by.render() }}) - from {{ tmp_relation }} - ); - - {# - TODO: include_sql_header is a hack; consider a better approach that includes - the sql_header at the materialization-level instead - #} - -- 3. run the merge statement - {{ get_insert_overwrite_merge_sql(target_relation, source_sql, dest_columns, [predicate], include_sql_header=false) }}; - - -- 4. clean up the temp table - drop table if exists {{ tmp_relation }} - - {% endif %} - -{% endmacro %} - - -{% macro bq_generate_incremental_build_sql( - strategy, tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists -) %} - {#-- if partitioned, use BQ scripting to get the range of partition values to be updated --#} - {% if strategy == 'insert_overwrite' %} - - {% set missing_partition_msg -%} - The 'insert_overwrite' strategy requires the `partition_by` config. - {%- endset %} - {% if partition_by is none %} - {% do exceptions.raise_compiler_error(missing_partition_msg) %} - {% endif %} - - {% set build_sql = bq_insert_overwrite( - tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, on_schema_change - ) %} - - {% else %} {# strategy == 'merge' #} - {%- set source_sql -%} - {%- if tmp_relation_exists -%} - ( - select * from {{ tmp_relation }} - ) - {%- else -%} {#-- wrap sql in parens to make it a subquery --#} - ( - {{sql}} - ) - {%- endif -%} - {%- endset -%} - - {% set build_sql = get_merge_sql(target_relation, source_sql, unique_key, dest_columns) %} - - {% endif %} - - {{ return(build_sql) }} - -{% endmacro %} - -{% materialization incremental, adapter='bigquery' -%} - - {%- set unique_key = config.get('unique_key') -%} - {%- set full_refresh_mode = (should_full_refresh()) -%} - - {%- set target_relation = this %} - {%- set existing_relation = load_relation(this) %} - {%- set tmp_relation = make_temp_relation(this) %} - - {#-- Validate early so we don't run SQL if the strategy is invalid --#} - {% set strategy = dbt_bigquery_validate_get_incremental_strategy(config) -%} - - {%- set raw_partition_by = config.get('partition_by', none) -%} - {%- set partition_by = adapter.parse_partition_by(raw_partition_by) -%} - {%- set partitions = config.get('partitions', none) -%} - {%- set cluster_by = config.get('cluster_by', none) -%} - - {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %} - - {{ run_hooks(pre_hooks) }} - - {% if existing_relation is none %} - {% set build_sql = create_table_as(False, target_relation, sql) %} - - {% elif existing_relation.is_view %} - {#-- There's no way to atomically replace a view with a table on BQ --#} - {{ adapter.drop_relation(existing_relation) }} - {% set build_sql = create_table_as(False, target_relation, sql) %} - - {% elif full_refresh_mode %} - {#-- If the partition/cluster config has changed, then we must drop and recreate --#} - {% if not adapter.is_replaceable(existing_relation, partition_by, cluster_by) %} - {% do log("Hard refreshing " ~ existing_relation ~ " because it is not replaceable") %} - {{ adapter.drop_relation(existing_relation) }} - {% endif %} - {% set build_sql = create_table_as(False, target_relation, sql) %} - - {% else %} - {% set tmp_relation_exists = false %} - {% if on_schema_change != 'ignore' %} {# Check first, since otherwise we may not build a temp table #} - {% do run_query(create_table_as(True, tmp_relation, sql)) %} - {% set tmp_relation_exists = true %} - {% do process_schema_changes(on_schema_change, tmp_relation, existing_relation) %} - {% endif %} - - {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %} - {% set build_sql = bq_generate_incremental_build_sql( - strategy, tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists - ) %} - - {% endif %} - - {%- call statement('main') -%} - {{ build_sql }} - {% endcall %} - - {{ run_hooks(post_hooks) }} - - {% set target_relation = this.incorporate(type='table') %} - - {% do persist_docs(target_relation, model) %} - - {{ return({'relations': [target_relation]}) }} - -{%- endmaterialization %} diff --git a/plugins/bigquery/dbt/include/bigquery/macros/materializations/seed.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/seed.sql deleted file mode 100644 index d95cc4e1b10..00000000000 --- a/plugins/bigquery/dbt/include/bigquery/macros/materializations/seed.sql +++ /dev/null @@ -1,19 +0,0 @@ - -{% macro bigquery__create_csv_table(model, agate_table) %} - -- no-op -{% endmacro %} - -{% macro bigquery__reset_csv_table(model, full_refresh, old_relation, agate_table) %} - {{ adapter.drop_relation(old_relation) }} -{% endmacro %} - -{% macro bigquery__load_csv_rows(model, agate_table) %} - - {%- set column_override = model['config'].get('column_types', {}) -%} - {{ adapter.load_dataframe(model['database'], model['schema'], model['alias'], - agate_table, column_override) }} - {% if config.persist_relation_docs() and 'description' in model %} - - {{ adapter.update_table_description(model['database'], model['schema'], model['alias'], model['description']) }} - {% endif %} -{% endmacro %} diff --git a/plugins/bigquery/dbt/include/bigquery/macros/materializations/snapshot.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/snapshot.sql deleted file mode 100644 index 836a44c8d72..00000000000 --- a/plugins/bigquery/dbt/include/bigquery/macros/materializations/snapshot.sql +++ /dev/null @@ -1,15 +0,0 @@ -{% macro bigquery__snapshot_hash_arguments(args) -%} - to_hex(md5(concat({%- for arg in args -%} - coalesce(cast({{ arg }} as string), ''){% if not loop.last %}, '|',{% endif -%} - {%- endfor -%} - ))) -{%- endmacro %} - -{% macro bigquery__create_columns(relation, columns) %} - {{ adapter.alter_table_add_columns(relation, columns) }} -{% endmacro %} - -{% macro bigquery__post_snapshot(staging_relation) %} - -- Clean up the snapshot temp table - {% do drop_relation(staging_relation) %} -{% endmacro %} diff --git a/plugins/bigquery/dbt/include/bigquery/macros/materializations/table.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/table.sql deleted file mode 100644 index bbfe547710e..00000000000 --- a/plugins/bigquery/dbt/include/bigquery/macros/materializations/table.sql +++ /dev/null @@ -1,84 +0,0 @@ -{% macro make_date_partitioned_table(model, relation, dates, should_create, verbose=False) %} - - {% if should_create %} - {{ adapter.make_date_partitioned_table(relation) }} - {% endif %} - - {% for date in dates %} - {% set date = (date | string) %} - {% if verbose %} - {% set table_start_time = modules.datetime.datetime.now().strftime("%H:%M:%S") %} - {{ log(table_start_time ~ ' | -> Running for day ' ~ date, info=True) }} - {% endif %} - - {% set fixed_sql = model['compiled_sql'] | replace('[DBT__PARTITION_DATE]', date) %} - {% set _ = adapter.execute_model(model, 'table', fixed_sql, decorator=date) %} - {% endfor %} - - {% set num_days = dates | length %} - {% if num_days == 1 %} - {% set result_str = 'CREATED 1 PARTITION' %} - {% else %} - {% set result_str = 'CREATED ' ~ num_days ~ ' PARTITIONS' %} - {% endif %} - - {{ store_result('main', response=result_str) }} - -{% endmacro %} - -{% materialization table, adapter='bigquery' -%} - - {%- set identifier = model['alias'] -%} - {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} - {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} - {%- set target_relation = api.Relation.create(database=database, schema=schema, identifier=identifier, type='table') -%} - {%- set verbose = config.get('verbose', False) -%} - - {# partitions: iterate over each partition, running a separate query in a for-loop #} - {%- set partitions = config.get('partitions') -%} - - {% if partitions %} - {% if partitions is number or partitions is string %} - {% set partitions = [(partitions | string)] %} - {% endif %} - - {% if partitions is not iterable %} - {{ exceptions.raise_compiler_error("Provided `partitions` configuration is not a list. Got: " ~ partitions, model) }} - {% endif %} - {% endif %} - - {{ run_hooks(pre_hooks) }} - - {# - Since dbt uses WRITE_TRUNCATE mode for tables, we only need to drop this thing - if it is not a table. If it _is_ already a table, then we can overwrite it without downtime - #} - {%- if exists_not_as_table -%} - {{ adapter.drop_relation(old_relation) }} - {%- endif -%} - - -- build model - {% if partitions %} - {# Create the dp-table if 1. it does not exist or 2. it existed, but we just dropped it #} - {%- set should_create = (old_relation is none or exists_not_as_table) -%} - {{ make_date_partitioned_table(model, target_relation, partitions, should_create, verbose) }} - {% else %} - {%- set raw_partition_by = config.get('partition_by', none) -%} - {%- set partition_by = adapter.parse_partition_by(raw_partition_by) -%} - {%- set cluster_by = config.get('cluster_by', none) -%} - {% if not adapter.is_replaceable(old_relation, partition_by, cluster_by) %} - {% do log("Hard refreshing " ~ old_relation ~ " because it is not replaceable") %} - {% do adapter.drop_relation(old_relation) %} - {% endif %} - {% call statement('main') -%} - {{ create_table_as(False, target_relation, sql) }} - {% endcall -%} - {% endif %} - - {{ run_hooks(post_hooks) }} - - {% do persist_docs(target_relation, model) %} - - {{ return({'relations': [target_relation]}) }} - -{% endmaterialization %} diff --git a/plugins/bigquery/dbt/include/bigquery/macros/materializations/view.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/view.sql deleted file mode 100644 index 97e3d2761fd..00000000000 --- a/plugins/bigquery/dbt/include/bigquery/macros/materializations/view.sql +++ /dev/null @@ -1,25 +0,0 @@ - -{% macro bigquery__handle_existing_table(full_refresh, old_relation) %} - {%- if full_refresh -%} - {{ adapter.drop_relation(old_relation) }} - {%- else -%} - {{ exceptions.relation_wrong_type(old_relation, 'view') }} - {%- endif -%} -{% endmacro %} - - -{% materialization view, adapter='bigquery' -%} - {% set to_return = create_or_replace_view() %} - - {% set target_relation = this.incorporate(type='view') %} - {% do persist_docs(target_relation, model) %} - - {% if config.get('grant_access_to') %} - {% for grant_target_dict in config.get('grant_access_to') %} - {% do adapter.grant_access_to(this, 'view', None, grant_target_dict) %} - {% endfor %} - {% endif %} - - {% do return(to_return) %} - -{%- endmaterialization %} diff --git a/plugins/bigquery/dbt/include/bigquery/sample_profiles.yml b/plugins/bigquery/dbt/include/bigquery/sample_profiles.yml deleted file mode 100644 index 36f47fdbcad..00000000000 --- a/plugins/bigquery/dbt/include/bigquery/sample_profiles.yml +++ /dev/null @@ -1,26 +0,0 @@ -default: - outputs: - - dev: - type: bigquery - method: oauth - project: [GCP project id] - dataset: [the name of your dbt dataset] # You can also use "schema" here - threads: [1 or more] - timeout_seconds: 300 - location: US # Optional, one of US or EU - priority: interactive - retries: 1 - - prod: - type: bigquery - method: service-account - project: [GCP project id] - dataset: [the name of your dbt dataset] - threads: [1 or more] - keyfile: [/path/to/bigquery/keyfile.json] - timeout_seconds: 300 - priority: interactive - retries: 1 - - target: dev \ No newline at end of file diff --git a/plugins/bigquery/setup.py b/plugins/bigquery/setup.py deleted file mode 100644 index 038033f5d0b..00000000000 --- a/plugins/bigquery/setup.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -if sys.version_info < (3, 6): - print('Error: dbt does not support this version of Python.') - print('Please upgrade to Python 3.6 or higher.') - sys.exit(1) - - -from setuptools import setup -try: - from setuptools import find_namespace_packages -except ImportError: - # the user has a downlevel version of setuptools. - print('Error: dbt requires setuptools v40.1.0 or higher.') - print('Please upgrade setuptools with "pip install --upgrade setuptools" ' - 'and try again') - sys.exit(1) - - -package_name = "dbt-bigquery" -package_version = "0.21.0rc1" -description = """The bigquery adapter plugin for dbt (data build tool)""" - -this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, 'README.md')) as f: - long_description = f.read() - -setup( - name=package_name, - version=package_version, - description=description, - long_description=long_description, - long_description_content_type='text/markdown', - author="dbt Labs", - author_email="info@dbtlabs.com", - url="https://github.com/dbt-labs/dbt", - packages=find_namespace_packages(include=['dbt', 'dbt.*']), - package_data={ - 'dbt': [ - 'include/bigquery/dbt_project.yml', - 'include/bigquery/sample_profiles.yml', - 'include/bigquery/macros/*.sql', - 'include/bigquery/macros/**/*.sql', - ] - }, - install_requires=[ - 'dbt-core=={}'.format(package_version), - 'protobuf>=3.13.0,<4', - 'google-cloud-core>=1.3.0,<2', - 'google-cloud-bigquery>=1.25.0,<3', - 'google-api-core>=1.16.0,<2', - 'googleapis-common-protos>=1.6.0,<2', - 'six>=1.14.0', - ], - zip_safe=False, - classifiers=[ - 'Development Status :: 5 - Production/Stable', - - 'License :: OSI Approved :: Apache Software License', - - 'Operating System :: Microsoft :: Windows', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: POSIX :: Linux', - - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - ], - python_requires=">=3.6.2", -) diff --git a/plugins/redshift/README.md b/plugins/redshift/README.md index f5152a205fe..0b57dace08e 100644 --- a/plugins/redshift/README.md +++ b/plugins/redshift/README.md @@ -1,32 +1,3 @@ -

- dbt logo -

+### dbt-redshift -**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications. - -dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis. - -## dbt-redshift - -The `dbt-redshift` package contains all of the code required to make dbt operate on a Redshift database. For -more information on using dbt with Redshift, consult [the docs](https://docs.getdbt.com/docs/profile-redshift). - - -## Find out more - -- Check out the [Introduction to dbt](https://docs.getdbt.com/docs/introduction/). -- Read the [dbt Viewpoint](https://docs.getdbt.com/docs/about/viewpoint/). - -## Join thousands of analysts in the dbt community - -- Join the [chat](http://community.getdbt.com/) on Slack. -- Find community posts on [dbt Discourse](https://discourse.getdbt.com). - -## Reporting bugs and contributing code - -- Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/dbt-labs/dbt/issues/new). -- Want to help us build dbt? Check out the [Contributing Getting Started Guide](https://github.com/dbt-labs/dbt/blob/HEAD/CONTRIBUTING.md) - -## Code of Conduct - -Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt Code of Conduct](https://community.getdbt.com/code-of-conduct). +This plugin has moved! https://github.com/dbt-labs/dbt-redshift \ No newline at end of file diff --git a/plugins/redshift/dbt/adapters/redshift/__init__.py b/plugins/redshift/dbt/adapters/redshift/__init__.py deleted file mode 100644 index f8059e1168b..00000000000 --- a/plugins/redshift/dbt/adapters/redshift/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from dbt.adapters.redshift.connections import RedshiftConnectionManager # noqa -from dbt.adapters.redshift.connections import RedshiftCredentials -from dbt.adapters.redshift.relation import RedshiftColumn # noqa -from dbt.adapters.redshift.relation import RedshiftRelation # noqa: F401 -from dbt.adapters.redshift.impl import RedshiftAdapter - - -from dbt.adapters.base import AdapterPlugin -from dbt.include import redshift - -Plugin = AdapterPlugin( - adapter=RedshiftAdapter, - credentials=RedshiftCredentials, - include_path=redshift.PACKAGE_PATH, - dependencies=['postgres']) diff --git a/plugins/redshift/dbt/adapters/redshift/__version__.py b/plugins/redshift/dbt/adapters/redshift/__version__.py deleted file mode 100644 index cebffb39326..00000000000 --- a/plugins/redshift/dbt/adapters/redshift/__version__.py +++ /dev/null @@ -1 +0,0 @@ -version = '0.21.0rc1' diff --git a/plugins/redshift/dbt/adapters/redshift/connections.py b/plugins/redshift/dbt/adapters/redshift/connections.py deleted file mode 100644 index 00864de065a..00000000000 --- a/plugins/redshift/dbt/adapters/redshift/connections.py +++ /dev/null @@ -1,174 +0,0 @@ -from multiprocessing import Lock -from contextlib import contextmanager -from typing import NewType - -from dbt.adapters.postgres import PostgresConnectionManager -from dbt.adapters.postgres import PostgresCredentials -from dbt.logger import GLOBAL_LOGGER as logger # noqa -import dbt.exceptions -import dbt.flags - -import boto3 - -from dbt.dataclass_schema import FieldEncoder, dbtClassMixin, StrEnum - -from dataclasses import dataclass, field -from typing import Optional, List - -drop_lock: Lock = dbt.flags.MP_CONTEXT.Lock() - - -IAMDuration = NewType('IAMDuration', int) - - -class IAMDurationEncoder(FieldEncoder): - @property - def json_schema(self): - return {'type': 'integer', 'minimum': 0, 'maximum': 65535} - - -dbtClassMixin.register_field_encoders({IAMDuration: IAMDurationEncoder()}) - - -class RedshiftConnectionMethod(StrEnum): - DATABASE = 'database' - IAM = 'iam' - - -@dataclass -class RedshiftCredentials(PostgresCredentials): - method: RedshiftConnectionMethod = RedshiftConnectionMethod.DATABASE - password: Optional[str] = None - cluster_id: Optional[str] = field( - default=None, - metadata={'description': 'If using IAM auth, the name of the cluster'}, - ) - iam_profile: Optional[str] = None - iam_duration_seconds: int = 900 - search_path: Optional[str] = None - keepalives_idle: int = 240 - autocreate: bool = False - db_groups: List[str] = field(default_factory=list) - ra3_node: Optional[bool] = False - - @property - def type(self): - return 'redshift' - - def _connection_keys(self): - keys = super()._connection_keys() - return keys + ( - 'method', - 'cluster_id', - 'iam_profile', - 'iam_duration_seconds' - ) - - -class RedshiftConnectionManager(PostgresConnectionManager): - TYPE = 'redshift' - - @contextmanager - def fresh_transaction(self, name=None): - """On entrance to this context manager, hold an exclusive lock and - create a fresh transaction for redshift, then commit and begin a new - one before releasing the lock on exit. - - See drop_relation in RedshiftAdapter for more information. - - :param Optional[str] name: The name of the connection to use, or None - to use the default. - """ - with drop_lock: - connection = self.get_thread_connection() - - if connection.transaction_open: - self.commit() - - self.begin() - yield - - self.commit() - self.begin() - - @classmethod - def fetch_cluster_credentials(cls, db_user, db_name, cluster_id, - iam_profile, duration_s, autocreate, - db_groups): - """Fetches temporary login credentials from AWS. The specified user - must already exist in the database, or else an error will occur""" - - if iam_profile is None: - session = boto3.Session() - boto_client = session.client("redshift") - else: - logger.debug("Connecting to Redshift using 'IAM'" + - f"with profile {iam_profile}") - boto_session = boto3.Session( - profile_name=iam_profile - ) - boto_client = boto_session.client('redshift') - - try: - return boto_client.get_cluster_credentials( - DbUser=db_user, - DbName=db_name, - ClusterIdentifier=cluster_id, - DurationSeconds=duration_s, - AutoCreate=autocreate, - DbGroups=db_groups,) - - except boto_client.exceptions.ClientError as e: - raise dbt.exceptions.FailedToConnectException( - "Unable to get temporary Redshift cluster credentials: {}" - .format(e)) - - @classmethod - def get_tmp_iam_cluster_credentials(cls, credentials): - cluster_id = credentials.cluster_id - - # default via: - # boto3.readthedocs.io/en/latest/reference/services/redshift.html - iam_duration_s = credentials.iam_duration_seconds - - if not cluster_id: - raise dbt.exceptions.FailedToConnectException( - "'cluster_id' must be provided in profile if IAM " - "authentication method selected") - - cluster_creds = cls.fetch_cluster_credentials( - credentials.user, - credentials.database, - credentials.cluster_id, - credentials.iam_profile, - iam_duration_s, - credentials.autocreate, - credentials.db_groups, - ) - - # replace username and password with temporary redshift credentials - return credentials.replace(user=cluster_creds.get('DbUser'), - password=cluster_creds.get('DbPassword')) - - @classmethod - def get_credentials(cls, credentials): - method = credentials.method - - # Support missing 'method' for backwards compatibility - if method == 'database' or method is None: - logger.debug("Connecting to Redshift using 'database' credentials") - # this requirement is really annoying to encode into json schema, - # so validate it here - if credentials.password is None: - raise dbt.exceptions.FailedToConnectException( - "'password' field is required for 'database' credentials" - ) - return credentials - - elif method == 'iam': - logger.debug("Connecting to Redshift using 'IAM' credentials") - return cls.get_tmp_iam_cluster_credentials(credentials) - - else: - raise dbt.exceptions.FailedToConnectException( - "Invalid 'method' in profile: '{}'".format(method)) diff --git a/plugins/redshift/dbt/adapters/redshift/impl.py b/plugins/redshift/dbt/adapters/redshift/impl.py deleted file mode 100644 index 284363c12e1..00000000000 --- a/plugins/redshift/dbt/adapters/redshift/impl.py +++ /dev/null @@ -1,88 +0,0 @@ -from dataclasses import dataclass -from typing import Optional -from dbt.adapters.base.impl import AdapterConfig -from dbt.adapters.sql import SQLAdapter -from dbt.adapters.base.meta import available -from dbt.adapters.postgres import PostgresAdapter -from dbt.adapters.redshift import RedshiftConnectionManager -from dbt.adapters.redshift import RedshiftColumn -from dbt.adapters.redshift import RedshiftRelation -from dbt.logger import GLOBAL_LOGGER as logger # noqa -import dbt.exceptions - - -@dataclass -class RedshiftConfig(AdapterConfig): - sort_type: Optional[str] = None - dist: Optional[str] = None - sort: Optional[str] = None - bind: Optional[bool] = None - - -class RedshiftAdapter(PostgresAdapter, SQLAdapter): - Relation = RedshiftRelation - ConnectionManager = RedshiftConnectionManager - Column = RedshiftColumn - - AdapterSpecificConfigs = RedshiftConfig - - @classmethod - def date_function(cls): - return 'getdate()' - - def drop_relation(self, relation): - """ - In Redshift, DROP TABLE ... CASCADE should not be used - inside a transaction. Redshift doesn't prevent the CASCADE - part from conflicting with concurrent transactions. If we do - attempt to drop two tables with CASCADE at once, we'll often - get the dreaded: - - table was dropped by a concurrent transaction - - So, we need to lock around calls to the underlying - drop_relation() function. - - https://docs.aws.amazon.com/redshift/latest/dg/r_DROP_TABLE.html - """ - with self.connections.fresh_transaction(): - return super().drop_relation(relation) - - @classmethod - def convert_text_type(cls, agate_table, col_idx): - column = agate_table.columns[col_idx] - # `lens` must be a list, so this can't be a generator expression, - # because max() raises ane exception if its argument has no members. - lens = [len(d.encode("utf-8")) for d in column.values_without_nulls()] - max_len = max(lens) if lens else 64 - return "varchar({})".format(max_len) - - @classmethod - def convert_time_type(cls, agate_table, col_idx): - return "varchar(24)" - - @available - def verify_database(self, database): - if database.startswith('"'): - database = database.strip('"') - expected = self.config.credentials.database - ra3_node = self.config.credentials.ra3_node - - if database.lower() != expected.lower() and not ra3_node: - raise dbt.exceptions.NotImplementedException( - 'Cross-db references allowed only in RA3.* node. ({} vs {})' - .format(database, expected) - ) - # return an empty string on success so macros can call this - return '' - - def _get_catalog_schemas(self, manifest): - # redshift(besides ra3) only allow one database (the main one) - schemas = super(SQLAdapter, self)._get_catalog_schemas(manifest) - try: - return schemas.flatten(allow_multiple_databases=self.config.credentials.ra3_node) - except dbt.exceptions.RuntimeException as exc: - dbt.exceptions.raise_compiler_error( - 'Cross-db references allowed only in {} RA3.* node. Got {}' - .format(self.type(), exc.msg) - ) diff --git a/plugins/redshift/dbt/adapters/redshift/relation.py b/plugins/redshift/dbt/adapters/redshift/relation.py deleted file mode 100644 index ac3ecdb9534..00000000000 --- a/plugins/redshift/dbt/adapters/redshift/relation.py +++ /dev/null @@ -1,15 +0,0 @@ -from dbt.adapters.base import Column -from dataclasses import dataclass -from dbt.adapters.postgres.relation import PostgresRelation - - -@dataclass(frozen=True, eq=False, repr=False) -class RedshiftRelation(PostgresRelation): - # Override the method in the Postgres Relation - # because Redshift allows longer names - def relation_max_name_length(self): - return 127 - - -class RedshiftColumn(Column): - pass # redshift does not inherit from postgres here diff --git a/plugins/redshift/dbt/include/redshift/__init__.py b/plugins/redshift/dbt/include/redshift/__init__.py deleted file mode 100644 index b177e5d4932..00000000000 --- a/plugins/redshift/dbt/include/redshift/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -import os - -PACKAGE_PATH = os.path.dirname(__file__) diff --git a/plugins/redshift/dbt/include/redshift/dbt_project.yml b/plugins/redshift/dbt/include/redshift/dbt_project.yml deleted file mode 100644 index 1efdab2c1b0..00000000000 --- a/plugins/redshift/dbt/include/redshift/dbt_project.yml +++ /dev/null @@ -1,5 +0,0 @@ -config-version: 2 -name: dbt_redshift -version: 1.0 - -macro-paths: ["macros"] diff --git a/plugins/redshift/dbt/include/redshift/macros/adapters.sql b/plugins/redshift/dbt/include/redshift/macros/adapters.sql deleted file mode 100644 index 95f92811cbe..00000000000 --- a/plugins/redshift/dbt/include/redshift/macros/adapters.sql +++ /dev/null @@ -1,283 +0,0 @@ - -{% macro dist(dist) %} - {%- if dist is not none -%} - {%- set dist = dist.strip().lower() -%} - - {%- if dist in ['all', 'even'] -%} - diststyle {{ dist }} - {%- elif dist == "auto" -%} - {%- else -%} - diststyle key distkey ({{ dist }}) - {%- endif -%} - - {%- endif -%} -{%- endmacro -%} - - -{% macro sort(sort_type, sort) %} - {%- if sort is not none %} - {{ sort_type | default('compound', boolean=true) }} sortkey( - {%- if sort is string -%} - {%- set sort = [sort] -%} - {%- endif -%} - {%- for item in sort -%} - {{ item }} - {%- if not loop.last -%},{%- endif -%} - {%- endfor -%} - ) - {%- endif %} -{%- endmacro -%} - - -{% macro redshift__create_table_as(temporary, relation, sql) -%} - - {%- set _dist = config.get('dist') -%} - {%- set _sort_type = config.get( - 'sort_type', - validator=validation.any['compound', 'interleaved']) -%} - {%- set _sort = config.get( - 'sort', - validator=validation.any[list, basestring]) -%} - {%- set sql_header = config.get('sql_header', none) -%} - - {{ sql_header if sql_header is not none }} - - create {% if temporary -%}temporary{%- endif %} table - {{ relation.include(database=(not temporary), schema=(not temporary)) }} - {{ dist(_dist) }} - {{ sort(_sort_type, _sort) }} - as ( - {{ sql }} - ); -{%- endmacro %} - - -{% macro redshift__create_view_as(relation, sql) -%} - {%- set binding = config.get('bind', default=True) -%} - - {% set bind_qualifier = '' if binding else 'with no schema binding' %} - {%- set sql_header = config.get('sql_header', none) -%} - - {{ sql_header if sql_header is not none }} - - create view {{ relation }} as ( - {{ sql }} - ) {{ bind_qualifier }}; -{% endmacro %} - - -{% macro redshift__create_schema(relation) -%} - {{ postgres__create_schema(relation) }} -{% endmacro %} - - -{% macro redshift__drop_schema(relation) -%} - {{ postgres__drop_schema(relation) }} -{% endmacro %} - - -{% macro redshift__get_columns_in_relation(relation) -%} - {% call statement('get_columns_in_relation', fetch_result=True) %} - with bound_views as ( - select - ordinal_position, - table_schema, - column_name, - data_type, - character_maximum_length, - numeric_precision, - numeric_scale - - from information_schema."columns" - where table_name = '{{ relation.identifier }}' - ), - - unbound_views as ( - select - ordinal_position, - view_schema, - col_name, - case - when col_type ilike 'character varying%' then - 'character varying' - when col_type ilike 'numeric%' then 'numeric' - else col_type - end as col_type, - case - when col_type like 'character%' - then nullif(REGEXP_SUBSTR(col_type, '[0-9]+'), '')::int - else null - end as character_maximum_length, - case - when col_type like 'numeric%' - then nullif( - SPLIT_PART(REGEXP_SUBSTR(col_type, '[0-9,]+'), ',', 1), - '')::int - else null - end as numeric_precision, - case - when col_type like 'numeric%' - then nullif( - SPLIT_PART(REGEXP_SUBSTR(col_type, '[0-9,]+'), ',', 2), - '')::int - else null - end as numeric_scale - - from pg_get_late_binding_view_cols() - cols(view_schema name, view_name name, col_name name, - col_type varchar, ordinal_position int) - where view_name = '{{ relation.identifier }}' - ), - - external_views as ( - select - columnnum, - schemaname, - columnname, - case - when external_type ilike 'character varying%' or external_type ilike 'varchar%' - then 'character varying' - when external_type ilike 'numeric%' then 'numeric' - else external_type - end as external_type, - case - when external_type like 'character%' or external_type like 'varchar%' - then nullif( - REGEXP_SUBSTR(external_type, '[0-9]+'), - '')::int - else null - end as character_maximum_length, - case - when external_type like 'numeric%' - then nullif( - SPLIT_PART(REGEXP_SUBSTR(external_type, '[0-9,]+'), ',', 1), - '')::int - else null - end as numeric_precision, - case - when external_type like 'numeric%' - then nullif( - SPLIT_PART(REGEXP_SUBSTR(external_type, '[0-9,]+'), ',', 2), - '')::int - else null - end as numeric_scale - from - pg_catalog.svv_external_columns - where - schemaname = '{{ relation.schema }}' - and tablename = '{{ relation.identifier }}' - - ), - - unioned as ( - select * from bound_views - union all - select * from unbound_views - union all - select * from external_views - ) - - select - column_name, - data_type, - character_maximum_length, - numeric_precision, - numeric_scale - - from unioned - {% if relation.schema %} - where table_schema = '{{ relation.schema }}' - {% endif %} - order by ordinal_position - {% endcall %} - {% set table = load_result('get_columns_in_relation').table %} - {{ return(sql_convert_columns_in_relation(table)) }} -{% endmacro %} - - -{% macro redshift__list_relations_without_caching(schema_relation) %} - {{ return(postgres__list_relations_without_caching(schema_relation)) }} -{% endmacro %} - - -{% macro redshift__information_schema_name(database) -%} - {{ return(postgres__information_schema_name(database)) }} -{%- endmacro %} - - -{% macro redshift__list_schemas(database) -%} - {{ return(postgres__list_schemas(database)) }} -{%- endmacro %} - - -{% macro redshift__check_schema_exists(information_schema, schema) -%} - {{ return(postgres__check_schema_exists(information_schema, schema)) }} -{%- endmacro %} - -{% macro redshift__current_timestamp() -%} - getdate() -{%- endmacro %} - -{% macro redshift__snapshot_get_time() -%} - {{ current_timestamp() }}::timestamp -{%- endmacro %} - - -{% macro redshift__snapshot_string_as_time(timestamp) -%} - {%- set result = "'" ~ timestamp ~ "'::timestamp" -%} - {{ return(result) }} -{%- endmacro %} - -{% macro redshift__make_temp_relation(base_relation, suffix) %} - {% do return(postgres__make_temp_relation(base_relation, suffix)) %} -{% endmacro %} - - -{% macro redshift__persist_docs(relation, model, for_relation, for_columns) -%} - {% if for_relation and config.persist_relation_docs() and model.description %} - {% do run_query(alter_relation_comment(relation, model.description)) %} - {% endif %} - - {# Override: do not set column comments for LBVs #} - {% set is_lbv = config.get('materialized') == 'view' and config.get('bind') == false %} - {% if for_columns and config.persist_column_docs() and model.columns and not is_lbv %} - {% do run_query(alter_column_comment(relation, model.columns)) %} - {% endif %} -{% endmacro %} - - -{% macro redshift__alter_relation_comment(relation, comment) %} - {% do return(postgres__alter_relation_comment(relation, comment)) %} -{% endmacro %} - - -{% macro redshift__alter_column_comment(relation, column_dict) %} - {% do return(postgres__alter_column_comment(relation, column_dict)) %} -{% endmacro %} - - -{% macro redshift__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %} - - {% if add_columns %} - - {% for column in add_columns %} - {% set sql -%} - alter {{ relation.type }} {{ relation }} add column {{ column.name }} {{ column.data_type }} - {% endset %} - {% do run_query(sql) %} - {% endfor %} - - {% endif %} - - {% if remove_columns %} - - {% for column in remove_columns %} - {% set sql -%} - alter {{ relation.type }} {{ relation }} drop column {{ column.name }} - {% endset %} - {% do run_query(sql) %} - {% endfor %} - - {% endif %} - -{% endmacro %} diff --git a/plugins/redshift/dbt/include/redshift/macros/catalog.sql b/plugins/redshift/dbt/include/redshift/macros/catalog.sql deleted file mode 100644 index b34460035e2..00000000000 --- a/plugins/redshift/dbt/include/redshift/macros/catalog.sql +++ /dev/null @@ -1,242 +0,0 @@ - -{% macro redshift__get_base_catalog(information_schema, schemas) -%} - {%- call statement('base_catalog', fetch_result=True) -%} - {% set database = information_schema.database %} - {{ adapter.verify_database(database) }} - - with late_binding as ( - select - '{{ database }}'::varchar as table_database, - table_schema, - table_name, - 'LATE BINDING VIEW'::varchar as table_type, - null::text as table_comment, - - column_name, - column_index, - column_type, - null::text as column_comment - from pg_get_late_binding_view_cols() - cols(table_schema name, table_name name, column_name name, - column_type varchar, - column_index int) - order by "column_index" - ), - - early_binding as ( - select - '{{ database }}'::varchar as table_database, - sch.nspname as table_schema, - tbl.relname as table_name, - case tbl.relkind - when 'v' then 'VIEW' - else 'BASE TABLE' - end as table_type, - tbl_desc.description as table_comment, - col.attname as column_name, - col.attnum as column_index, - pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type, - col_desc.description as column_comment - - from pg_catalog.pg_namespace sch - join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid - join pg_catalog.pg_attribute col on col.attrelid = tbl.oid - left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0) - left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum) - where ( - {%- for schema in schemas -%} - upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - and tbl.relkind in ('r', 'v', 'f', 'p') - and col.attnum > 0 - and not col.attisdropped - ), - - table_owners as ( - - select - '{{ database }}'::varchar as table_database, - schemaname as table_schema, - tablename as table_name, - tableowner as table_owner - - from pg_tables - - union all - - select - '{{ database }}'::varchar as table_database, - schemaname as table_schema, - viewname as table_name, - viewowner as table_owner - - from pg_views - - ), - - unioned as ( - - select * - from early_binding - - union all - - select * - from late_binding - - ) - - select *, - table_database || '.' || table_schema || '.' || table_name as table_id - - from unioned - join table_owners using (table_database, table_schema, table_name) - - where ( - {%- for schema in schemas -%} - upper(table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - - order by "column_index" - {%- endcall -%} - - {{ return(load_result('base_catalog').table) }} -{%- endmacro %} - -{% macro redshift__get_extended_catalog(schemas) %} - {%- call statement('extended_catalog', fetch_result=True) -%} - - select - "database" || '.' || "schema" || '.' || "table" as table_id, - - 'Encoded'::text as "stats:encoded:label", - encoded as "stats:encoded:value", - 'Indicates whether any column in the table has compression encoding defined.'::text as "stats:encoded:description", - true as "stats:encoded:include", - - 'Dist Style' as "stats:diststyle:label", - diststyle as "stats:diststyle:value", - 'Distribution style or distribution key column, if key distribution is defined.'::text as "stats:diststyle:description", - true as "stats:diststyle:include", - - 'Sort Key 1' as "stats:sortkey1:label", - -- handle 0xFF byte in response for interleaved sort styles - case - when sortkey1 like 'INTERLEAVED%' then 'INTERLEAVED'::text - else sortkey1 - end as "stats:sortkey1:value", - 'First column in the sort key.'::text as "stats:sortkey1:description", - (sortkey1 is not null) as "stats:sortkey1:include", - - 'Max Varchar' as "stats:max_varchar:label", - max_varchar as "stats:max_varchar:value", - 'Size of the largest column that uses a VARCHAR data type.'::text as "stats:max_varchar:description", - true as "stats:max_varchar:include", - - -- exclude this, as the data is strangely returned with null-byte characters - 'Sort Key 1 Encoding' as "stats:sortkey1_enc:label", - sortkey1_enc as "stats:sortkey1_enc:value", - 'Compression encoding of the first column in the sort key.' as "stats:sortkey1_enc:description", - false as "stats:sortkey1_enc:include", - - '# Sort Keys' as "stats:sortkey_num:label", - sortkey_num as "stats:sortkey_num:value", - 'Number of columns defined as sort keys.' as "stats:sortkey_num:description", - (sortkey_num > 0) as "stats:sortkey_num:include", - - 'Approximate Size' as "stats:size:label", - size * 1000000 as "stats:size:value", - 'Approximate size of the table, calculated from a count of 1MB blocks'::text as "stats:size:description", - true as "stats:size:include", - - 'Disk Utilization' as "stats:pct_used:label", - pct_used / 100.0 as "stats:pct_used:value", - 'Percent of available space that is used by the table.'::text as "stats:pct_used:description", - true as "stats:pct_used:include", - - 'Unsorted %' as "stats:unsorted:label", - unsorted / 100.0 as "stats:unsorted:value", - 'Percent of unsorted rows in the table.'::text as "stats:unsorted:description", - (unsorted is not null) as "stats:unsorted:include", - - 'Stats Off' as "stats:stats_off:label", - stats_off as "stats:stats_off:value", - 'Number that indicates how stale the table statistics are; 0 is current, 100 is out of date.'::text as "stats:stats_off:description", - true as "stats:stats_off:include", - - 'Approximate Row Count' as "stats:rows:label", - tbl_rows as "stats:rows:value", - 'Approximate number of rows in the table. This value includes rows marked for deletion, but not yet vacuumed.'::text as "stats:rows:description", - true as "stats:rows:include", - - 'Sort Key Skew' as "stats:skew_sortkey1:label", - skew_sortkey1 as "stats:skew_sortkey1:value", - 'Ratio of the size of the largest non-sort key column to the size of the first column of the sort key.'::text as "stats:skew_sortkey1:description", - (skew_sortkey1 is not null) as "stats:skew_sortkey1:include", - - 'Skew Rows' as "stats:skew_rows:label", - skew_rows as "stats:skew_rows:value", - 'Ratio of the number of rows in the slice with the most rows to the number of rows in the slice with the fewest rows.'::text as "stats:skew_rows:description", - (skew_rows is not null) as "stats:skew_rows:include" - - from svv_table_info - where ( - {%- for schema in schemas -%} - upper(schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - - {%- endcall -%} - - {{ return(load_result('extended_catalog').table) }} - -{% endmacro %} - -{% macro redshift__can_select_from(table_name) %} - - {%- call statement('has_table_privilege', fetch_result=True) -%} - - select has_table_privilege(current_user, '{{ table_name }}', 'SELECT') as can_select - - {%- endcall -%} - - {% set can_select = load_result('has_table_privilege').table[0]['can_select'] %} - {{ return(can_select) }} - -{% endmacro %} - -{% macro redshift__no_svv_table_info_warning() %} - - {% set msg %} - - Warning: The database user "{{ target.user }}" has insufficient permissions to - query the "svv_table_info" table. Please grant SELECT permissions on this table - to the "{{ target.user }}" user to fetch extended table details from Redshift. - - {% endset %} - - {{ log(msg, info=True) }} - -{% endmacro %} - - -{% macro redshift__get_catalog(information_schema, schemas) %} - - {#-- Compute a left-outer join in memory. Some Redshift queries are - -- leader-only, and cannot be joined to other compute-based queries #} - - {% set catalog = redshift__get_base_catalog(information_schema, schemas) %} - - {% set select_extended = redshift__can_select_from('svv_table_info') %} - {% if select_extended %} - {% set extended_catalog = redshift__get_extended_catalog(schemas) %} - {% set catalog = catalog.join(extended_catalog, 'table_id') %} - {% else %} - {{ redshift__no_svv_table_info_warning() }} - {% endif %} - - {{ return(catalog.exclude(['table_id'])) }} - -{% endmacro %} diff --git a/plugins/redshift/dbt/include/redshift/macros/materializations/snapshot_merge.sql b/plugins/redshift/dbt/include/redshift/macros/materializations/snapshot_merge.sql deleted file mode 100644 index eda31472733..00000000000 --- a/plugins/redshift/dbt/include/redshift/macros/materializations/snapshot_merge.sql +++ /dev/null @@ -1,4 +0,0 @@ - -{% macro redshift__snapshot_merge_sql(target, source, insert_cols) -%} - {{ postgres__snapshot_merge_sql(target, source, insert_cols) }} -{% endmacro %} diff --git a/plugins/redshift/dbt/include/redshift/macros/relations.sql b/plugins/redshift/dbt/include/redshift/macros/relations.sql deleted file mode 100644 index ed682ae7d63..00000000000 --- a/plugins/redshift/dbt/include/redshift/macros/relations.sql +++ /dev/null @@ -1,3 +0,0 @@ -{% macro redshift__get_relations () -%} - {{ return(dbt.postgres__get_relations()) }} -{% endmacro %} diff --git a/plugins/redshift/dbt/include/redshift/sample_profiles.yml b/plugins/redshift/dbt/include/redshift/sample_profiles.yml deleted file mode 100644 index 46cd979344c..00000000000 --- a/plugins/redshift/dbt/include/redshift/sample_profiles.yml +++ /dev/null @@ -1,25 +0,0 @@ -default: - outputs: - - dev: - type: redshift - threads: [1 or more] - host: [host] - port: [port] - user: [dev_username] - pass: [dev_password] - dbname: [dbname] - schema: [dev_schema] - - prod: - type: redshift - method: iam - cluster_id: [cluster_id] - threads: [1 or more] - host: [host] - port: [port] - user: [prod_user] - dbname: [dbname] - schema: [prod_schema] - - target: dev diff --git a/plugins/redshift/setup.py b/plugins/redshift/setup.py deleted file mode 100644 index 4ff761ca152..00000000000 --- a/plugins/redshift/setup.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -if sys.version_info < (3, 6): - print('Error: dbt does not support this version of Python.') - print('Please upgrade to Python 3.6 or higher.') - sys.exit(1) - - -from setuptools import setup -try: - from setuptools import find_namespace_packages -except ImportError: - # the user has a downlevel version of setuptools. - print('Error: dbt requires setuptools v40.1.0 or higher.') - print('Please upgrade setuptools with "pip install --upgrade setuptools" ' - 'and try again') - sys.exit(1) - - -package_name = "dbt-redshift" -package_version = "0.21.0rc1" -description = """The redshift adapter plugin for dbt (data build tool)""" - -this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, 'README.md')) as f: - long_description = f.read() - -setup( - name=package_name, - version=package_version, - description=description, - long_description=description, - long_description_content_type='text/markdown', - author="dbt Labs", - author_email="info@dbtlabs.com", - url="https://github.com/dbt-labs/dbt", - packages=find_namespace_packages(include=['dbt', 'dbt.*']), - package_data={ - 'dbt': [ - 'include/redshift/dbt_project.yml', - 'include/redshift/sample_profiles.yml', - 'include/redshift/macros/*.sql', - 'include/redshift/macros/**/*.sql', - ] - }, - install_requires=[ - 'dbt-core=={}'.format(package_version), - 'dbt-postgres=={}'.format(package_version), - # the following are all to match snowflake-connector-python - 'boto3>=1.4.4,<2.0.0', - ], - zip_safe=False, - classifiers=[ - 'Development Status :: 5 - Production/Stable', - - 'License :: OSI Approved :: Apache Software License', - - 'Operating System :: Microsoft :: Windows', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: POSIX :: Linux', - - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - ], - python_requires=">=3.6.2", -) diff --git a/plugins/snowflake/README.md b/plugins/snowflake/README.md index 784419bae0b..b68cf734f1b 100644 --- a/plugins/snowflake/README.md +++ b/plugins/snowflake/README.md @@ -1,32 +1,3 @@ -

- dbt logo -

+### dbt-snowflake -**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications. - -dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis. - -## dbt-snowflake - -The `dbt-snowflake` package contains all of the code required to make dbt operate on a Snowflake database. For -more information on using dbt with Snowflake, consult [the docs](https://docs.getdbt.com/docs/profile-snowflake). - - -## Find out more - -- Check out the [Introduction to dbt](https://docs.getdbt.com/docs/introduction/). -- Read the [dbt Viewpoint](https://docs.getdbt.com/docs/about/viewpoint/). - -## Join thousands of analysts in the dbt community - -- Join the [chat](http://community.getdbt.com/) on Slack. -- Find community posts on [dbt Discourse](https://discourse.getdbt.com). - -## Reporting bugs and contributing code - -- Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/dbt-labs/dbt/issues/new). -- Want to help us build dbt? Check out the [Contributing Getting Started Guide](https://github.com/dbt-labs/dbt/blob/HEAD/CONTRIBUTING.md) - -## Code of Conduct - -Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt Code of Conduct](https://community.getdbt.com/code-of-conduct). +This plugin has moved! https://github.com/dbt-labs/dbt-snowflake \ No newline at end of file diff --git a/plugins/snowflake/dbt/adapters/snowflake/__init__.py b/plugins/snowflake/dbt/adapters/snowflake/__init__.py deleted file mode 100644 index bb4e17c9bb3..00000000000 --- a/plugins/snowflake/dbt/adapters/snowflake/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from dbt.adapters.snowflake.column import SnowflakeColumn # noqa -from dbt.adapters.snowflake.connections import SnowflakeConnectionManager # noqa -from dbt.adapters.snowflake.connections import SnowflakeCredentials -from dbt.adapters.snowflake.relation import SnowflakeRelation # noqa -from dbt.adapters.snowflake.impl import SnowflakeAdapter - -from dbt.adapters.base import AdapterPlugin -from dbt.include import snowflake - -Plugin = AdapterPlugin( - adapter=SnowflakeAdapter, - credentials=SnowflakeCredentials, - include_path=snowflake.PACKAGE_PATH) diff --git a/plugins/snowflake/dbt/adapters/snowflake/__version__.py b/plugins/snowflake/dbt/adapters/snowflake/__version__.py deleted file mode 100644 index cebffb39326..00000000000 --- a/plugins/snowflake/dbt/adapters/snowflake/__version__.py +++ /dev/null @@ -1 +0,0 @@ -version = '0.21.0rc1' diff --git a/plugins/snowflake/dbt/adapters/snowflake/column.py b/plugins/snowflake/dbt/adapters/snowflake/column.py deleted file mode 100644 index d7afb307fac..00000000000 --- a/plugins/snowflake/dbt/adapters/snowflake/column.py +++ /dev/null @@ -1,31 +0,0 @@ -from dataclasses import dataclass - -from dbt.adapters.base.column import Column -from dbt.exceptions import RuntimeException - - -@dataclass -class SnowflakeColumn(Column): - def is_integer(self) -> bool: - # everything that smells like an int is actually a NUMBER(38, 0) - return False - - def is_numeric(self) -> bool: - return self.dtype.lower() in [ - 'int', 'integer', 'bigint', 'smallint', 'tinyint', 'byteint', - 'numeric', 'decimal', 'number' - ] - - def is_float(self): - return self.dtype.lower() in [ - 'float', 'float4', 'float8', 'double', 'double precision', 'real', - ] - - def string_size(self) -> int: - if not self.is_string(): - raise RuntimeException("Called string_size() on non-string field!") - - if self.dtype == 'text' or self.char_size is None: - return 16777216 - else: - return int(self.char_size) diff --git a/plugins/snowflake/dbt/adapters/snowflake/connections.py b/plugins/snowflake/dbt/adapters/snowflake/connections.py deleted file mode 100644 index b92ff029d27..00000000000 --- a/plugins/snowflake/dbt/adapters/snowflake/connections.py +++ /dev/null @@ -1,375 +0,0 @@ -import base64 -import datetime -import pytz -import re -from contextlib import contextmanager -from dataclasses import dataclass -from io import StringIO -from time import sleep -from typing import Optional - -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization -import requests -import snowflake.connector -import snowflake.connector.errors - -from dbt.exceptions import ( - InternalException, RuntimeException, FailedToConnectException, - DatabaseException, warn_or_error -) -from dbt.adapters.base import Credentials -from dbt.contracts.connection import AdapterResponse -from dbt.adapters.sql import SQLConnectionManager -from dbt.logger import GLOBAL_LOGGER as logger - - -_TOKEN_REQUEST_URL = 'https://{}.snowflakecomputing.com/oauth/token-request' - - -@dataclass -class SnowflakeCredentials(Credentials): - account: str - user: str - warehouse: Optional[str] = None - role: Optional[str] = None - password: Optional[str] = None - authenticator: Optional[str] = None - private_key_path: Optional[str] = None - private_key_passphrase: Optional[str] = None - token: Optional[str] = None - oauth_client_id: Optional[str] = None - oauth_client_secret: Optional[str] = None - query_tag: Optional[str] = None - client_session_keep_alive: bool = False - - def __post_init__(self): - if ( - self.authenticator != 'oauth' and - (self.oauth_client_secret or self.oauth_client_id or self.token) - ): - # the user probably forgot to set 'authenticator' like I keep doing - warn_or_error( - 'Authenticator is not set to oauth, but an oauth-only ' - 'parameter is set! Did you mean to set authenticator: oauth?' - ) - - @property - def type(self): - return 'snowflake' - - @property - def unique_field(self): - return self.account - - def _connection_keys(self): - return ( - 'account', 'user', 'database', 'schema', 'warehouse', 'role', - 'client_session_keep_alive' - ) - - def auth_args(self): - # Pull all of the optional authentication args for the connector, - # let connector handle the actual arg validation - result = {} - if self.password: - result['password'] = self.password - if self.authenticator: - result['authenticator'] = self.authenticator - if self.authenticator == 'oauth': - token = self.token - # if we have a client ID/client secret, the token is a refresh - # token, not an access token - if self.oauth_client_id and self.oauth_client_secret: - token = self._get_access_token() - elif self.oauth_client_id: - warn_or_error( - 'Invalid profile: got an oauth_client_id, but not an ' - 'oauth_client_secret!' - ) - elif self.oauth_client_secret: - warn_or_error( - 'Invalid profile: got an oauth_client_secret, but not ' - 'an oauth_client_id!' - ) - - result['token'] = token - # enable the token cache - result['client_store_temporary_credential'] = True - result['private_key'] = self._get_private_key() - return result - - def _get_access_token(self) -> str: - if self.authenticator != 'oauth': - raise InternalException('Can only get access tokens for oauth') - missing = any( - x is None for x in - (self.oauth_client_id, self.oauth_client_secret, self.token) - ) - if missing: - raise InternalException( - 'need a client ID a client secret, and a refresh token to get ' - 'an access token' - ) - - # should the full url be a config item? - token_url = _TOKEN_REQUEST_URL.format(self.account) - # I think this is only used to redirect on success, which we ignore - # (it does not have to match the integration's settings in snowflake) - redirect_uri = 'http://localhost:9999' - data = { - 'grant_type': 'refresh_token', - 'refresh_token': self.token, - 'redirect_uri': redirect_uri - } - - auth = base64.b64encode( - f'{self.oauth_client_id}:{self.oauth_client_secret}' - .encode('ascii') - ).decode('ascii') - headers = { - 'Authorization': f'Basic {auth}', - 'Content-type': 'application/x-www-form-urlencoded;charset=utf-8' - } - - result_json = None - max_iter = 20 - # Attempt to obtain JSON for 1 second before throwing an error - for i in range(max_iter): - result = requests.post(token_url, headers=headers, data=data) - try: - result_json = result.json() - break - except ValueError as e: - message = result.text - logger.debug(f"Got a non-json response ({result.status_code}): \ - {e}, message: {message}") - sleep(0.05) - - if result_json is None: - raise DatabaseException(f"""Did not receive valid json with access_token. - Showing json response: {result_json}""") - - return result_json['access_token'] - - def _get_private_key(self): - """Get Snowflake private key by path or None.""" - if not self.private_key_path: - return None - - if self.private_key_passphrase: - encoded_passphrase = self.private_key_passphrase.encode() - else: - encoded_passphrase = None - - with open(self.private_key_path, 'rb') as key: - p_key = serialization.load_pem_private_key( - key.read(), - password=encoded_passphrase, - backend=default_backend()) - - return p_key.private_bytes( - encoding=serialization.Encoding.DER, - format=serialization.PrivateFormat.PKCS8, - encryption_algorithm=serialization.NoEncryption()) - - -class SnowflakeConnectionManager(SQLConnectionManager): - TYPE = 'snowflake' - - @contextmanager - def exception_handler(self, sql): - try: - yield - except snowflake.connector.errors.ProgrammingError as e: - msg = str(e) - - logger.debug('Snowflake query id: {}'.format(e.sfqid)) - logger.debug('Snowflake error: {}'.format(msg)) - - if 'Empty SQL statement' in msg: - logger.debug("got empty sql statement, moving on") - elif 'This session does not have a current database' in msg: - raise FailedToConnectException( - ('{}\n\nThis error sometimes occurs when invalid ' - 'credentials are provided, or when your default role ' - 'does not have access to use the specified database. ' - 'Please double check your profile and try again.') - .format(msg)) - else: - raise DatabaseException(msg) - except Exception as e: - if isinstance(e, snowflake.connector.errors.Error): - logger.debug('Snowflake query id: {}'.format(e.sfqid)) - - logger.debug("Error running SQL: {}", sql) - logger.debug("Rolling back transaction.") - self.rollback_if_open() - if isinstance(e, RuntimeException): - # during a sql query, an internal to dbt exception was raised. - # this sounds a lot like a signal handler and probably has - # useful information, so raise it without modification. - raise - raise RuntimeException(str(e)) from e - - @classmethod - def open(cls, connection): - if connection.state == 'open': - logger.debug('Connection is already open, skipping open.') - return connection - - try: - creds = connection.credentials - - handle = snowflake.connector.connect( - account=creds.account, - user=creds.user, - database=creds.database, - schema=creds.schema, - warehouse=creds.warehouse, - role=creds.role, - autocommit=True, - client_session_keep_alive=creds.client_session_keep_alive, - application='dbt', - **creds.auth_args() - ) - - if creds.query_tag: - handle.cursor().execute( - ("alter session set query_tag = '{}'") - .format(creds.query_tag)) - - connection.handle = handle - connection.state = 'open' - except snowflake.connector.errors.Error as e: - logger.debug("Got an error when attempting to open a snowflake " - "connection: '{}'" - .format(e)) - - connection.handle = None - connection.state = 'fail' - - raise FailedToConnectException(str(e)) - - def cancel(self, connection): - handle = connection.handle - sid = handle.session_id - - connection_name = connection.name - - sql = 'select system$abort_session({})'.format(sid) - - logger.debug("Cancelling query '{}' ({})".format(connection_name, sid)) - - _, cursor = self.add_query(sql) - res = cursor.fetchone() - - logger.debug("Cancel query '{}': {}".format(connection_name, res)) - - @classmethod - def get_response(cls, cursor) -> AdapterResponse: - code = cursor.sqlstate - - if code is None: - code = 'SUCCESS' - - return AdapterResponse( - _message="{} {}".format(code, cursor.rowcount), - rows_affected=cursor.rowcount, - code=code - ) - - # disable transactional logic by default on Snowflake - # except for DML statements where explicitly defined - def add_begin_query(self, *args, **kwargs): - pass - - def add_commit_query(self, *args, **kwargs): - pass - - def begin(self): - pass - - def commit(self): - pass - - def clear_transaction(self): - pass - - @classmethod - def _split_queries(cls, sql): - "Splits sql statements at semicolons into discrete queries" - - sql_s = str(sql) - sql_buf = StringIO(sql_s) - split_query = snowflake.connector.util_text.split_statements(sql_buf) - return [part[0] for part in split_query] - - @classmethod - def process_results(cls, column_names, rows): - # Override for Snowflake. The datetime objects returned by - # snowflake-connector-python are not pickleable, so we need - # to replace them with sane timezones - fixed = [] - for row in rows: - fixed_row = [] - for col in row: - if isinstance(col, datetime.datetime) and col.tzinfo: - offset = col.utcoffset() - offset_seconds = offset.total_seconds() - new_timezone = pytz.FixedOffset(offset_seconds // 60) - col = col.astimezone(tz=new_timezone) - fixed_row.append(col) - - fixed.append(fixed_row) - - return super().process_results(column_names, fixed) - - def add_query(self, sql, auto_begin=True, - bindings=None, abridge_sql_log=False): - - connection = None - cursor = None - - if bindings: - # The snowflake connector is more strict than, eg., psycopg2 - - # which allows any iterable thing to be passed as a binding. - bindings = tuple(bindings) - - queries = self._split_queries(sql) - - for individual_query in queries: - # hack -- after the last ';', remove comments and don't run - # empty queries. this avoids using exceptions as flow control, - # and also allows us to return the status of the last cursor - without_comments = re.sub( - re.compile( - r'(\".*?\"|\'.*?\')|(/\*.*?\*/|--[^\r\n]*$)', re.MULTILINE - ), - '', individual_query).strip() - - if without_comments == "": - continue - - connection, cursor = super().add_query( - individual_query, auto_begin, - bindings=bindings, - abridge_sql_log=abridge_sql_log - ) - - if cursor is None: - conn = self.get_thread_connection() - if conn is None or conn.name is None: - conn_name = '' - else: - conn_name = conn.name - - raise RuntimeException( - "Tried to run an empty query on model '{}'. If you are " - "conditionally running\nsql, eg. in a model hook, make " - "sure your `else` clause contains valid sql!\n\n" - "Provided SQL:\n{}" - .format(conn_name, sql) - ) - - return connection, cursor diff --git a/plugins/snowflake/dbt/adapters/snowflake/impl.py b/plugins/snowflake/dbt/adapters/snowflake/impl.py deleted file mode 100644 index e609ba5f52f..00000000000 --- a/plugins/snowflake/dbt/adapters/snowflake/impl.py +++ /dev/null @@ -1,190 +0,0 @@ -from dataclasses import dataclass -from typing import Mapping, Any, Optional, List, Union - -import agate - -from dbt.adapters.base.impl import AdapterConfig -from dbt.adapters.sql import SQLAdapter -from dbt.adapters.sql.impl import ( - LIST_SCHEMAS_MACRO_NAME, - LIST_RELATIONS_MACRO_NAME, -) -from dbt.adapters.snowflake import SnowflakeConnectionManager -from dbt.adapters.snowflake import SnowflakeRelation -from dbt.adapters.snowflake import SnowflakeColumn -from dbt.contracts.graph.manifest import Manifest -from dbt.exceptions import ( - raise_compiler_error, RuntimeException, DatabaseException -) -from dbt.utils import filter_null_values - - -@dataclass -class SnowflakeConfig(AdapterConfig): - transient: Optional[bool] = None - cluster_by: Optional[Union[str, List[str]]] = None - automatic_clustering: Optional[bool] = None - secure: Optional[bool] = None - copy_grants: Optional[bool] = None - snowflake_warehouse: Optional[str] = None - query_tag: Optional[str] = None - merge_update_columns: Optional[str] = None - - -class SnowflakeAdapter(SQLAdapter): - Relation = SnowflakeRelation - Column = SnowflakeColumn - ConnectionManager = SnowflakeConnectionManager - - AdapterSpecificConfigs = SnowflakeConfig - - @classmethod - def date_function(cls): - return "CURRENT_TIMESTAMP()" - - @classmethod - def _catalog_filter_table( - cls, table: agate.Table, manifest: Manifest - ) -> agate.Table: - # On snowflake, users can set QUOTED_IDENTIFIERS_IGNORE_CASE, so force - # the column names to their lowercased forms. - lowered = table.rename( - column_names=[c.lower() for c in table.column_names] - ) - return super()._catalog_filter_table(lowered, manifest) - - def _make_match_kwargs(self, database, schema, identifier): - quoting = self.config.quoting - if identifier is not None and quoting["identifier"] is False: - identifier = identifier.upper() - - if schema is not None and quoting["schema"] is False: - schema = schema.upper() - - if database is not None and quoting["database"] is False: - database = database.upper() - - return filter_null_values( - {"identifier": identifier, "schema": schema, "database": database} - ) - - def _get_warehouse(self) -> str: - _, table = self.execute( - 'select current_warehouse() as warehouse', - fetch=True - ) - if len(table) == 0 or len(table[0]) == 0: - # can this happen? - raise RuntimeException( - 'Could not get current warehouse: no results' - ) - return str(table[0][0]) - - def _use_warehouse(self, warehouse: str): - """Use the given warehouse. Quotes are never applied.""" - self.execute('use warehouse {}'.format(warehouse)) - - def pre_model_hook(self, config: Mapping[str, Any]) -> Optional[str]: - default_warehouse = self.config.credentials.warehouse - warehouse = config.get('snowflake_warehouse', default_warehouse) - if warehouse == default_warehouse or warehouse is None: - return None - previous = self._get_warehouse() - self._use_warehouse(warehouse) - return previous - - def post_model_hook( - self, config: Mapping[str, Any], context: Optional[str] - ) -> None: - if context is not None: - self._use_warehouse(context) - - def list_schemas(self, database: str) -> List[str]: - try: - results = self.execute_macro( - LIST_SCHEMAS_MACRO_NAME, - kwargs={'database': database} - ) - except DatabaseException as exc: - msg = ( - f'Database error while listing schemas in database ' - f'"{database}"\n{exc}' - ) - raise RuntimeException(msg) - # this uses 'show terse schemas in database', and the column name we - # want is 'name' - - return [row['name'] for row in results] - - def get_columns_in_relation(self, relation): - try: - return super().get_columns_in_relation(relation) - except DatabaseException as exc: - if 'does not exist or not authorized' in str(exc): - return [] - else: - raise - - def list_relations_without_caching( - self, schema_relation: SnowflakeRelation - ) -> List[SnowflakeRelation]: - kwargs = {'schema_relation': schema_relation} - try: - results = self.execute_macro( - LIST_RELATIONS_MACRO_NAME, - kwargs=kwargs - ) - except DatabaseException as exc: - # if the schema doesn't exist, we just want to return. - # Alternatively, we could query the list of schemas before we start - # and skip listing the missing ones, which sounds expensive. - if 'Object does not exist' in str(exc): - return [] - raise - - relations = [] - quote_policy = { - 'database': True, - 'schema': True, - 'identifier': True - } - - columns = ['database_name', 'schema_name', 'name', 'kind'] - for _database, _schema, _identifier, _type in results.select(columns): - try: - _type = self.Relation.get_relation_type(_type.lower()) - except ValueError: - _type = self.Relation.External - relations.append(self.Relation.create( - database=_database, - schema=_schema, - identifier=_identifier, - quote_policy=quote_policy, - type=_type - )) - - return relations - - def quote_seed_column( - self, column: str, quote_config: Optional[bool] - ) -> str: - quote_columns: bool = False - if isinstance(quote_config, bool): - quote_columns = quote_config - elif quote_config is None: - pass - else: - raise_compiler_error( - f'The seed configuration value of "quote_columns" has an ' - f'invalid type {type(quote_config)}' - ) - - if quote_columns: - return self.quote(column) - else: - return column - - def timestamp_add_sql( - self, add_to: str, number: int = 1, interval: str = 'hour' - ) -> str: - return f'DATEADD({interval}, {number}, {add_to})' diff --git a/plugins/snowflake/dbt/adapters/snowflake/relation.py b/plugins/snowflake/dbt/adapters/snowflake/relation.py deleted file mode 100644 index 217292d8d17..00000000000 --- a/plugins/snowflake/dbt/adapters/snowflake/relation.py +++ /dev/null @@ -1,14 +0,0 @@ -from dataclasses import dataclass -from dbt.adapters.base.relation import BaseRelation, Policy - - -@dataclass -class SnowflakeQuotePolicy(Policy): - database: bool = False - schema: bool = False - identifier: bool = False - - -@dataclass(frozen=True, eq=False, repr=False) -class SnowflakeRelation(BaseRelation): - quote_policy: SnowflakeQuotePolicy = SnowflakeQuotePolicy() diff --git a/plugins/snowflake/dbt/include/snowflake/__init__.py b/plugins/snowflake/dbt/include/snowflake/__init__.py deleted file mode 100644 index 564a3d1e80a..00000000000 --- a/plugins/snowflake/dbt/include/snowflake/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -import os -PACKAGE_PATH = os.path.dirname(__file__) diff --git a/plugins/snowflake/dbt/include/snowflake/dbt_project.yml b/plugins/snowflake/dbt/include/snowflake/dbt_project.yml deleted file mode 100644 index fcd2c9a4822..00000000000 --- a/plugins/snowflake/dbt/include/snowflake/dbt_project.yml +++ /dev/null @@ -1,5 +0,0 @@ -config-version: 2 -name: dbt_snowflake -version: 1.0 - -macro-paths: ["macros"] diff --git a/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql b/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql deleted file mode 100644 index 2d184536985..00000000000 --- a/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql +++ /dev/null @@ -1,251 +0,0 @@ -{% macro snowflake__create_table_as(temporary, relation, sql) -%} - {%- set transient = config.get('transient', default=true) -%} - {%- set cluster_by_keys = config.get('cluster_by', default=none) -%} - {%- set enable_automatic_clustering = config.get('automatic_clustering', default=false) -%} - {%- set copy_grants = config.get('copy_grants', default=false) -%} - - {%- if cluster_by_keys is not none and cluster_by_keys is string -%} - {%- set cluster_by_keys = [cluster_by_keys] -%} - {%- endif -%} - {%- if cluster_by_keys is not none -%} - {%- set cluster_by_string = cluster_by_keys|join(", ")-%} - {% else %} - {%- set cluster_by_string = none -%} - {%- endif -%} - {%- set sql_header = config.get('sql_header', none) -%} - - {{ sql_header if sql_header is not none }} - - create or replace {% if temporary -%} - temporary - {%- elif transient -%} - transient - {%- endif %} table {{ relation }} {% if copy_grants and not temporary -%} copy grants {%- endif %} as - ( - {%- if cluster_by_string is not none -%} - select * from( - {{ sql }} - ) order by ({{ cluster_by_string }}) - {%- else -%} - {{ sql }} - {%- endif %} - ); - {% if cluster_by_string is not none and not temporary -%} - alter table {{relation}} cluster by ({{cluster_by_string}}); - {%- endif -%} - {% if enable_automatic_clustering and cluster_by_string is not none and not temporary -%} - alter table {{relation}} resume recluster; - {%- endif -%} -{% endmacro %} - -{% macro snowflake__create_view_as(relation, sql) -%} - {%- set secure = config.get('secure', default=false) -%} - {%- set copy_grants = config.get('copy_grants', default=false) -%} - {%- set sql_header = config.get('sql_header', none) -%} - - {{ sql_header if sql_header is not none }} - create or replace {% if secure -%} - secure - {%- endif %} view {{ relation }} {% if copy_grants -%} copy grants {%- endif %} as ( - {{ sql }} - ); -{% endmacro %} - -{% macro snowflake__get_columns_in_relation(relation) -%} - {%- set sql -%} - describe table {{ relation }} - {%- endset -%} - {%- set result = run_query(sql) -%} - - {% set maximum = 10000 %} - {% if (result | length) >= maximum %} - {% set msg %} - Too many columns in relation {{ relation }}! dbt can only get - information about relations with fewer than {{ maximum }} columns. - {% endset %} - {% do exceptions.raise_compiler_error(msg) %} - {% endif %} - - {% set columns = [] %} - {% for row in result %} - {% do columns.append(api.Column.from_description(row['name'], row['type'])) %} - {% endfor %} - {% do return(columns) %} -{% endmacro %} - -{% macro snowflake__list_schemas(database) -%} - {# 10k limit from here: https://docs.snowflake.net/manuals/sql-reference/sql/show-schemas.html#usage-notes #} - {% set maximum = 10000 %} - {% set sql -%} - show terse schemas in database {{ database }} - limit {{ maximum }} - {%- endset %} - {% set result = run_query(sql) %} - {% if (result | length) >= maximum %} - {% set msg %} - Too many schemas in database {{ database }}! dbt can only get - information about databases with fewer than {{ maximum }} schemas. - {% endset %} - {% do exceptions.raise_compiler_error(msg) %} - {% endif %} - {{ return(result) }} -{% endmacro %} - - -{% macro snowflake__list_relations_without_caching(schema_relation) %} - {%- set sql -%} - show terse objects in {{ schema_relation }} - {%- endset -%} - - {%- set result = run_query(sql) -%} - {% set maximum = 10000 %} - {% if (result | length) >= maximum %} - {% set msg %} - Too many schemas in schema {{ schema_relation }}! dbt can only get - information about schemas with fewer than {{ maximum }} objects. - {% endset %} - {% do exceptions.raise_compiler_error(msg) %} - {% endif %} - {%- do return(result) -%} -{% endmacro %} - - -{% macro snowflake__check_schema_exists(information_schema, schema) -%} - {% call statement('check_schema_exists', fetch_result=True) -%} - select count(*) - from {{ information_schema }}.schemata - where upper(schema_name) = upper('{{ schema }}') - and upper(catalog_name) = upper('{{ information_schema.database }}') - {%- endcall %} - {{ return(load_result('check_schema_exists').table) }} -{%- endmacro %} - -{% macro snowflake__current_timestamp() -%} - convert_timezone('UTC', current_timestamp()) -{%- endmacro %} - - -{% macro snowflake__snapshot_string_as_time(timestamp) -%} - {%- set result = "to_timestamp_ntz('" ~ timestamp ~ "')" -%} - {{ return(result) }} -{%- endmacro %} - - -{% macro snowflake__snapshot_get_time() -%} - to_timestamp_ntz({{ current_timestamp() }}) -{%- endmacro %} - - -{% macro snowflake__rename_relation(from_relation, to_relation) -%} - {% call statement('rename_relation') -%} - alter table {{ from_relation }} rename to {{ to_relation }} - {%- endcall %} -{% endmacro %} - - -{% macro snowflake__alter_column_type(relation, column_name, new_column_type) -%} - {% call statement('alter_column_type') %} - alter table {{ relation }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }}; - {% endcall %} -{% endmacro %} - -{% macro snowflake__alter_relation_comment(relation, relation_comment) -%} - comment on {{ relation.type }} {{ relation }} IS $${{ relation_comment | replace('$', '[$]') }}$$; -{% endmacro %} - - -{% macro snowflake__alter_column_comment(relation, column_dict) -%} - {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %} - alter {{ relation.type }} {{ relation }} alter - {% for column_name in column_dict if (column_name in existing_columns) or (column_name|upper in existing_columns) %} - {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} COMMENT $${{ column_dict[column_name]['description'] | replace('$', '[$]') }}$$ {{ ',' if not loop.last else ';' }} - {% endfor %} -{% endmacro %} - - -{% macro get_current_query_tag() -%} - {{ return(run_query("show parameters like 'query_tag' in session").rows[0]['value']) }} -{% endmacro %} - - -{% macro set_query_tag() -%} - {% set new_query_tag = config.get('query_tag') %} - {% if new_query_tag %} - {% set original_query_tag = get_current_query_tag() %} - {{ log("Setting query_tag to '" ~ new_query_tag ~ "'. Will reset to '" ~ original_query_tag ~ "' after materialization.") }} - {% do run_query("alter session set query_tag = '{}'".format(new_query_tag)) %} - {{ return(original_query_tag)}} - {% endif %} - {{ return(none)}} -{% endmacro %} - -{% macro unset_query_tag(original_query_tag) -%} - {% set new_query_tag = config.get('query_tag') %} - {% if new_query_tag %} - {% if original_query_tag %} - {{ log("Resetting query_tag to '" ~ original_query_tag ~ "'.") }} - {% do run_query("alter session set query_tag = '{}'".format(original_query_tag)) %} - {% else %} - {{ log("No original query_tag, unsetting parameter.") }} - {% do run_query("alter session unset query_tag") %} - {% endif %} - {% endif %} -{% endmacro %} - - -{% macro snowflake__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %} - - {% if add_columns %} - - {% set sql -%} - alter {{ relation.type }} {{ relation }} add column - {% for column in add_columns %} - {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} - {% endfor %} - {%- endset -%} - - {% do run_query(sql) %} - - {% endif %} - - {% if remove_columns %} - - {% set sql -%} - alter {{ relation.type }} {{ relation }} drop column - {% for column in remove_columns %} - {{ column.name }}{{ ',' if not loop.last }} - {% endfor %} - {%- endset -%} - - {% do run_query(sql) %} - - {% endif %} - -{% endmacro %} - - -{% macro snowflake_dml_explicit_transaction(dml) %} - {# - Use this macro to wrap all INSERT, MERGE, UPDATE, DELETE, and TRUNCATE - statements before passing them into run_query(), or calling in the 'main' statement - of a materialization - #} - {% set dml_transaction -%} - begin; - {{ dml }}; - commit; - {%- endset %} - - {% do return(dml_transaction) %} - -{% endmacro %} - - -{% macro snowflake__truncate_relation(relation) -%} - {% set truncate_dml %} - truncate table {{ relation }} - {% endset %} - {% call statement('truncate_relation') -%} - {{ snowflake_dml_explicit_transaction(truncate_dml) }} - {%- endcall %} -{% endmacro %} diff --git a/plugins/snowflake/dbt/include/snowflake/macros/catalog.sql b/plugins/snowflake/dbt/include/snowflake/macros/catalog.sql deleted file mode 100644 index 51dedc2683c..00000000000 --- a/plugins/snowflake/dbt/include/snowflake/macros/catalog.sql +++ /dev/null @@ -1,67 +0,0 @@ -{% macro snowflake__get_catalog(information_schema, schemas) -%} - {% set query %} - with tables as ( - - select - table_catalog as "table_database", - table_schema as "table_schema", - table_name as "table_name", - table_type as "table_type", - comment as "table_comment", - - -- note: this is the _role_ that owns the table - table_owner as "table_owner", - - 'Clustering Key' as "stats:clustering_key:label", - clustering_key as "stats:clustering_key:value", - 'The key used to cluster this table' as "stats:clustering_key:description", - (clustering_key is not null) as "stats:clustering_key:include", - - 'Row Count' as "stats:row_count:label", - row_count as "stats:row_count:value", - 'An approximate count of rows in this table' as "stats:row_count:description", - (row_count is not null) as "stats:row_count:include", - - 'Approximate Size' as "stats:bytes:label", - bytes as "stats:bytes:value", - 'Approximate size of the table as reported by Snowflake' as "stats:bytes:description", - (bytes is not null) as "stats:bytes:include", - - 'Last Modified' as "stats:last_modified:label", - to_varchar(convert_timezone('UTC', last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", - 'The timestamp for last update/change' as "stats:last_modified:description", - (last_altered is not null and table_type='BASE TABLE') as "stats:last_modified:include" - - from {{ information_schema }}.tables - - ), - - columns as ( - - select - table_catalog as "table_database", - table_schema as "table_schema", - table_name as "table_name", - - column_name as "column_name", - ordinal_position as "column_index", - data_type as "column_type", - comment as "column_comment" - - from {{ information_schema }}.columns - ) - - select * - from tables - join columns using ("table_database", "table_schema", "table_name") - where ( - {%- for schema in schemas -%} - upper("table_schema") = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - order by "column_index" - {%- endset -%} - - {{ return(run_query(query)) }} - -{%- endmacro %} diff --git a/plugins/snowflake/dbt/include/snowflake/macros/materializations/incremental.sql b/plugins/snowflake/dbt/include/snowflake/macros/materializations/incremental.sql deleted file mode 100644 index 8c49ec5f0d6..00000000000 --- a/plugins/snowflake/dbt/include/snowflake/macros/materializations/incremental.sql +++ /dev/null @@ -1,80 +0,0 @@ - -{% macro dbt_snowflake_validate_get_incremental_strategy(config) %} - {#-- Find and validate the incremental strategy #} - {%- set strategy = config.get("incremental_strategy", default="merge") -%} - - {% set invalid_strategy_msg -%} - Invalid incremental strategy provided: {{ strategy }} - Expected one of: 'merge', 'delete+insert' - {%- endset %} - {% if strategy not in ['merge', 'delete+insert'] %} - {% do exceptions.raise_compiler_error(invalid_strategy_msg) %} - {% endif %} - - {% do return(strategy) %} -{% endmacro %} - -{% macro dbt_snowflake_get_incremental_sql(strategy, tmp_relation, target_relation, unique_key, dest_columns) %} - {% if strategy == 'merge' %} - {% do return(get_merge_sql(target_relation, tmp_relation, unique_key, dest_columns)) %} - {% elif strategy == 'delete+insert' %} - {% do return(get_delete_insert_merge_sql(target_relation, tmp_relation, unique_key, dest_columns)) %} - {% else %} - {% do exceptions.raise_compiler_error('invalid strategy: ' ~ strategy) %} - {% endif %} -{% endmacro %} - -{% materialization incremental, adapter='snowflake' -%} - - {% set original_query_tag = set_query_tag() %} - - {%- set unique_key = config.get('unique_key') -%} - {%- set full_refresh_mode = (should_full_refresh()) -%} - - {% set target_relation = this %} - {% set existing_relation = load_relation(this) %} - {% set tmp_relation = make_temp_relation(this) %} - - {#-- Validate early so we don't run SQL if the strategy is invalid --#} - {% set strategy = dbt_snowflake_validate_get_incremental_strategy(config) -%} - {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %} - - {{ run_hooks(pre_hooks) }} - - {% if existing_relation is none %} - {% set build_sql = create_table_as(False, target_relation, sql) %} - - {% elif existing_relation.is_view %} - {#-- Can't overwrite a view with a table - we must drop --#} - {{ log("Dropping relation " ~ target_relation ~ " because it is a view and this model is a table.") }} - {% do adapter.drop_relation(existing_relation) %} - {% set build_sql = create_table_as(False, target_relation, sql) %} - - {% elif full_refresh_mode %} - {% set build_sql = create_table_as(False, target_relation, sql) %} - - {% else %} - {% do run_query(create_table_as(True, tmp_relation, sql)) %} - {% do adapter.expand_target_column_types( - from_relation=tmp_relation, - to_relation=target_relation) %} - {% do process_schema_changes(on_schema_change, tmp_relation, existing_relation) %} - {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %} - {% set build_sql = dbt_snowflake_get_incremental_sql(strategy, tmp_relation, target_relation, unique_key, dest_columns) %} - - {% endif %} - - {%- call statement('main') -%} - {{ build_sql }} - {%- endcall -%} - - {{ run_hooks(post_hooks) }} - - {% set target_relation = target_relation.incorporate(type='table') %} - {% do persist_docs(target_relation, model) %} - - {% do unset_query_tag(original_query_tag) %} - - {{ return({'relations': [target_relation]}) }} - -{%- endmaterialization %} \ No newline at end of file diff --git a/plugins/snowflake/dbt/include/snowflake/macros/materializations/merge.sql b/plugins/snowflake/dbt/include/snowflake/macros/materializations/merge.sql deleted file mode 100644 index 6a8f6b89c04..00000000000 --- a/plugins/snowflake/dbt/include/snowflake/macros/materializations/merge.sql +++ /dev/null @@ -1,44 +0,0 @@ -{% macro snowflake__get_merge_sql(target, source_sql, unique_key, dest_columns, predicates) -%} - - {# - Workaround for Snowflake not being happy with a merge on a constant-false predicate. - When no unique_key is provided, this macro will do a regular insert. If a unique_key - is provided, then this macro will do a proper merge instead. - #} - - {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute='name')) -%} - {%- set sql_header = config.get('sql_header', none) -%} - - {%- set dml -%} - {%- if unique_key is none -%} - - {{ sql_header if sql_header is not none }} - - insert into {{ target }} ({{ dest_cols_csv }}) - ( - select {{ dest_cols_csv }} - from {{ source_sql }} - ) - - {%- else -%} - - {{ default__get_merge_sql(target, source_sql, unique_key, dest_columns, predicates) }} - - {%- endif -%} - {%- endset -%} - - {% do return(snowflake_dml_explicit_transaction(dml)) %} - -{% endmacro %} - - -{% macro snowflake__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) %} - {% set dml = default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) %} - {% do return(snowflake_dml_explicit_transaction(dml)) %} -{% endmacro %} - - -{% macro snowflake__snapshot_merge_sql(target, source, insert_cols) %} - {% set dml = default__snapshot_merge_sql(target, source, insert_cols) %} - {% do return(snowflake_dml_explicit_transaction(dml)) %} -{% endmacro %} diff --git a/plugins/snowflake/dbt/include/snowflake/macros/materializations/seed.sql b/plugins/snowflake/dbt/include/snowflake/macros/materializations/seed.sql deleted file mode 100644 index ad9c65c4a4a..00000000000 --- a/plugins/snowflake/dbt/include/snowflake/macros/materializations/seed.sql +++ /dev/null @@ -1,37 +0,0 @@ -{% macro snowflake__load_csv_rows(model, agate_table) %} - {% set batch_size = get_batch_size() %} - {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %} - {% set bindings = [] %} - - {% set statements = [] %} - - {% for chunk in agate_table.rows | batch(batch_size) %} - {% set bindings = [] %} - - {% for row in chunk %} - {% do bindings.extend(row) %} - {% endfor %} - - {% set sql %} - insert into {{ this.render() }} ({{ cols_sql }}) values - {% for row in chunk -%} - ({%- for column in agate_table.column_names -%} - %s - {%- if not loop.last%},{%- endif %} - {%- endfor -%}) - {%- if not loop.last%},{%- endif %} - {%- endfor %} - {% endset %} - - {% do adapter.add_query('BEGIN', auto_begin=False) %} - {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %} - {% do adapter.add_query('COMMIT', auto_begin=False) %} - - {% if loop.index0 == 0 %} - {% do statements.append(sql) %} - {% endif %} - {% endfor %} - - {# Return SQL so we can render it out into the compiled files #} - {{ return(statements[0]) }} -{% endmacro %} diff --git a/plugins/snowflake/dbt/include/snowflake/macros/materializations/table.sql b/plugins/snowflake/dbt/include/snowflake/macros/materializations/table.sql deleted file mode 100644 index 49f97069b8e..00000000000 --- a/plugins/snowflake/dbt/include/snowflake/macros/materializations/table.sql +++ /dev/null @@ -1,34 +0,0 @@ -{% materialization table, adapter='snowflake' %} - - {% set original_query_tag = set_query_tag() %} - - {%- set identifier = model['alias'] -%} - - {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} - {%- set target_relation = api.Relation.create(identifier=identifier, - schema=schema, - database=database, type='table') -%} - - {{ run_hooks(pre_hooks) }} - - {#-- Drop the relation if it was a view to "convert" it in a table. This may lead to - -- downtime, but it should be a relatively infrequent occurrence #} - {% if old_relation is not none and not old_relation.is_table %} - {{ log("Dropping relation " ~ old_relation ~ " because it is of type " ~ old_relation.type) }} - {{ drop_relation_if_exists(old_relation) }} - {% endif %} - - --build model - {% call statement('main') -%} - {{ create_table_as(false, target_relation, sql) }} - {%- endcall %} - - {{ run_hooks(post_hooks) }} - - {% do persist_docs(target_relation, model) %} - - {% do unset_query_tag(original_query_tag) %} - - {{ return({'relations': [target_relation]}) }} - -{% endmaterialization %} diff --git a/plugins/snowflake/dbt/include/snowflake/macros/materializations/view.sql b/plugins/snowflake/dbt/include/snowflake/macros/materializations/view.sql deleted file mode 100644 index b3858c67478..00000000000 --- a/plugins/snowflake/dbt/include/snowflake/macros/materializations/view.sql +++ /dev/null @@ -1,13 +0,0 @@ -{% materialization view, adapter='snowflake' -%} - - {% set original_query_tag = set_query_tag() %} - {% set to_return = create_or_replace_view() %} - - {% set target_relation = this.incorporate(type='view') %} - {% do persist_docs(target_relation, model, for_columns=false) %} - - {% do return(to_return) %} - - {% do unset_query_tag(original_query_tag) %} - -{%- endmaterialization %} diff --git a/plugins/snowflake/dbt/include/snowflake/sample_profiles.yml b/plugins/snowflake/dbt/include/snowflake/sample_profiles.yml deleted file mode 100644 index 22dca47c586..00000000000 --- a/plugins/snowflake/dbt/include/snowflake/sample_profiles.yml +++ /dev/null @@ -1,29 +0,0 @@ -default: - outputs: - - dev: # User-Password config - type: snowflake - account: [account id + region (if applicable)] - user: [username] - password: [password] - role: [user role] - database: [database name] - warehouse: [warehouse name] - schema: [dbt schema] - threads: [1 or more] - client_session_keep_alive: False - - prod: # Keypair config - type: snowflake - account: [account id + region (if applicable)] - user: [username] - role: [user role] - private_key_path: [path/to/private.key] - private_key_passphrase: [passphrase for the private key, if key is encrypted] - database: [database name] - warehouse: [warehouse name] - schema: [dbt schema] - threads: [1 or more] - client_session_keep_alive: False - - target: dev \ No newline at end of file diff --git a/plugins/snowflake/setup.py b/plugins/snowflake/setup.py deleted file mode 100644 index 5ec4c445f85..00000000000 --- a/plugins/snowflake/setup.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -if sys.version_info < (3, 6): - print('Error: dbt does not support this version of Python.') - print('Please upgrade to Python 3.6 or higher.') - sys.exit(1) - - -from setuptools import setup -try: - from setuptools import find_namespace_packages -except ImportError: - # the user has a downlevel version of setuptools. - print('Error: dbt requires setuptools v40.1.0 or higher.') - print('Please upgrade setuptools with "pip install --upgrade setuptools" ' - 'and try again') - sys.exit(1) - - -package_name = "dbt-snowflake" -package_version = "0.21.0rc1" -description = """The snowflake adapter plugin for dbt (data build tool)""" - -this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, 'README.md')) as f: - long_description = f.read() - -setup( - name=package_name, - version=package_version, - description=description, - long_description=description, - long_description_content_type='text/markdown', - author="dbt Labs", - author_email="info@dbtlabs.com", - url="https://github.com/dbt-labs/dbt", - packages=find_namespace_packages(include=['dbt', 'dbt.*']), - package_data={ - 'dbt': [ - 'include/snowflake/dbt_project.yml', - 'include/snowflake/sample_profiles.yml', - 'include/snowflake/macros/*.sql', - 'include/snowflake/macros/**/*.sql', - ] - }, - install_requires=[ - 'dbt-core=={}'.format(package_version), - 'snowflake-connector-python[secure-local-storage]>=2.4.1,<2.6.0', - 'requests<3.0.0', - 'cryptography>=3.2,<4', - ], - zip_safe=False, - classifiers=[ - 'Development Status :: 5 - Production/Stable', - - 'License :: OSI Approved :: Apache Software License', - - 'Operating System :: Microsoft :: Windows', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: POSIX :: Linux', - - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - ], - python_requires=">=3.6.2", -) diff --git a/requirements.txt b/requirements.txt index 9b82a9f79d9..279403c7e64 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,2 @@ ./core ./plugins/postgres -./plugins/redshift -./plugins/snowflake -./plugins/bigquery diff --git a/scripts/build-dist.sh b/scripts/build-dist.sh index 1cfe41a8c1f..5f04ef451ee 100755 --- a/scripts/build-dist.sh +++ b/scripts/build-dist.sh @@ -14,7 +14,7 @@ rm -rf "$DBT_PATH"/dist rm -rf "$DBT_PATH"/build mkdir -p "$DBT_PATH"/dist -for SUBPATH in core plugins/postgres plugins/redshift plugins/bigquery plugins/snowflake +for SUBPATH in core plugins/postgres do rm -rf "$DBT_PATH"/"$SUBPATH"/dist rm -rf "$DBT_PATH"/"$SUBPATH"/build diff --git a/setup.py b/setup.py index b3896839297..e66ede591fb 100644 --- a/setup.py +++ b/setup.py @@ -44,9 +44,6 @@ install_requires=[ 'dbt-core=={}'.format(package_version), 'dbt-postgres=={}'.format(package_version), - 'dbt-redshift=={}'.format(package_version), - 'dbt-snowflake=={}'.format(package_version), - 'dbt-bigquery=={}'.format(package_version), ], zip_safe=False, classifiers=[ diff --git a/test.env.sample b/test.env.sample deleted file mode 100644 index c31f2b88557..00000000000 --- a/test.env.sample +++ /dev/null @@ -1,22 +0,0 @@ -DBT_INVOCATION_ENV=development - -SNOWFLAKE_TEST_ACCOUNT= -SNOWFLAKE_TEST_USER= -SNOWFLAKE_TEST_PASSWORD= -SNOWFLAKE_TEST_DATABASE= -SNOWFLAKE_TEST_ALT_DATABASE= -SNOWFLAKE_TEST_QUOTED_DATABASE= -SNOWFLAKE_TEST_WAREHOUSE= -SNOWFLAKE_TEST_ALT_WAREHOUSE= -SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN= -SNOWFLAKE_TEST_OAUTH_CLIENT_ID= -SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET= - -BIGQUERY_TEST_SERVICE_ACCOUNT_JSON= -BIGQUERY_TEST_ALT_DATABASE= - -REDSHIFT_TEST_HOST= -REDSHIFT_TEST_USER= -REDSHIFT_TEST_PASS= -REDSHIFT_TEST_PORT= -REDSHIFT_TEST_DBNAME= diff --git a/test/integration/001_simple_copy_test/models-merge-update/incremental_update_cols.sql b/test/integration/001_simple_copy_test/models-merge-update/incremental_update_cols.sql deleted file mode 100644 index 6f79e08e9d7..00000000000 --- a/test/integration/001_simple_copy_test/models-merge-update/incremental_update_cols.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ - config( - materialized = "incremental", - unique_key = "id", - merge_update_columns = ["email", "ip_address"] - ) -}} - - -select * -from {{ ref('seed') }} - -{% if is_incremental() %} - - where load_date > (select max(load_date) from {{this}}) - -{% endif %} diff --git a/test/integration/001_simple_copy_test/models-snowflake/incremental_overwrite.sql b/test/integration/001_simple_copy_test/models-snowflake/incremental_overwrite.sql index aea98867512..ab1bf4c63de 100644 --- a/test/integration/001_simple_copy_test/models-snowflake/incremental_overwrite.sql +++ b/test/integration/001_simple_copy_test/models-snowflake/incremental_overwrite.sql @@ -1,9 +1,6 @@ {{ config(materialized='incremental', unique_key='id') }} --- this will fail on snowflake with "merge" due --- to the nondeterministic join on id - select 1 as id union all select 1 as id diff --git a/test/integration/001_simple_copy_test/seeds-merge-cols-initial/seed.csv b/test/integration/001_simple_copy_test/seeds-merge-cols-initial/seed.csv deleted file mode 100644 index 1333fd2028b..00000000000 --- a/test/integration/001_simple_copy_test/seeds-merge-cols-initial/seed.csv +++ /dev/null @@ -1,101 +0,0 @@ -load_date,id,first_name,last_name,email,gender,ip_address -2021-03-05,1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 -2021-03-05,2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 -2021-03-05,3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 -2021-03-05,4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 -2021-03-05,5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 -2021-03-05,6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 -2021-03-05,7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 -2021-03-05,8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 -2021-03-05,9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 -2021-03-05,10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 -2021-03-05,11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 -2021-03-05,12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 -2021-03-05,13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 -2021-03-05,14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 -2021-03-05,15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 -2021-03-05,16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 -2021-03-05,17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 -2021-03-05,18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 -2021-03-05,19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 -2021-03-05,20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 -2021-03-05,21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 -2021-03-05,22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 -2021-03-05,23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 -2021-03-05,24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 -2021-03-05,25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 -2021-03-05,26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 -2021-03-05,27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 -2021-03-05,28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 -2021-03-05,29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 -2021-03-05,30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 -2021-03-05,31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 -2021-03-05,32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 -2021-03-05,33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 -2021-03-05,34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 -2021-03-05,35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 -2021-03-05,36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 -2021-03-05,37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 -2021-03-05,38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 -2021-03-05,39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 -2021-03-05,40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 -2021-03-05,41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 -2021-03-05,42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 -2021-03-05,43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 -2021-03-05,44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 -2021-03-05,45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 -2021-03-05,46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 -2021-03-05,47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 -2021-03-05,48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 -2021-03-05,49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 -2021-03-05,50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 -2021-03-05,51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 -2021-03-05,52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 -2021-03-05,53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 -2021-03-05,54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 -2021-03-05,55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 -2021-03-05,56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 -2021-03-05,57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 -2021-03-05,58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 -2021-03-05,59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 -2021-03-05,60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 -2021-03-05,61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 -2021-03-05,62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 -2021-03-05,63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 -2021-03-05,64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 -2021-03-05,65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 -2021-03-05,66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 -2021-03-05,67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 -2021-03-05,68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 -2021-03-05,69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 -2021-03-05,70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 -2021-03-05,71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 -2021-03-05,72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 -2021-03-05,73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 -2021-03-05,74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 -2021-03-05,75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 -2021-03-05,76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 -2021-03-05,77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 -2021-03-05,78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 -2021-03-05,79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 -2021-03-05,80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 -2021-03-05,81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 -2021-03-05,82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 -2021-03-05,83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 -2021-03-05,84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 -2021-03-05,85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 -2021-03-05,86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 -2021-03-05,87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 -2021-03-05,88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 -2021-03-05,89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 -2021-03-05,90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 -2021-03-05,91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 -2021-03-05,92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 -2021-03-05,93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 -2021-03-05,94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 -2021-03-05,95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 -2021-03-05,96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 -2021-03-05,97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 -2021-03-05,98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 -2021-03-05,99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 -2021-03-05,100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 diff --git a/test/integration/001_simple_copy_test/seeds-merge-cols-update/expected_result.csv b/test/integration/001_simple_copy_test/seeds-merge-cols-update/expected_result.csv deleted file mode 100644 index b9e0e900b8c..00000000000 --- a/test/integration/001_simple_copy_test/seeds-merge-cols-update/expected_result.csv +++ /dev/null @@ -1,201 +0,0 @@ -load_date,id,first_name,last_name,email,gender,ip_address -2021-03-05,1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 -2021-03-05,2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 -2021-03-05,3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 -2021-03-05,4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 -2021-03-05,5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 -2021-03-05,6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 -2021-03-05,7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 -2021-03-05,8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 -2021-03-05,9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 -2021-03-05,10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 -2021-03-05,11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 -2021-03-05,12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 -2021-03-05,13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 -2021-03-05,14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 -2021-03-05,15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 -2021-03-05,16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 -2021-03-05,17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 -2021-03-05,18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 -2021-03-05,19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 -2021-03-05,20,Andrew,Davis,adavisj@reddit.com,Male,9.255.67.25 -2021-03-05,21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 -2021-03-05,22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 -2021-03-05,23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 -2021-03-05,24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 -2021-03-05,25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 -2021-03-05,26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 -2021-03-05,27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 -2021-03-05,28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 -2021-03-05,29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 -2021-03-05,30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 -2021-03-05,31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 -2021-03-05,32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 -2021-03-05,33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 -2021-03-05,34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 -2021-03-05,35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 -2021-03-05,36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 -2021-03-05,37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 -2021-03-05,38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 -2021-03-05,39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 -2021-03-05,40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 -2021-03-05,41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 -2021-03-05,42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 -2021-03-05,43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 -2021-03-05,44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 -2021-03-05,45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 -2021-03-05,46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 -2021-03-05,47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 -2021-03-05,48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 -2021-03-05,49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 -2021-03-05,50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 -2021-03-05,51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 -2021-03-05,52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 -2021-03-05,53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 -2021-03-05,54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 -2021-03-05,55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 -2021-03-05,56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 -2021-03-05,57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 -2021-03-05,58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 -2021-03-05,59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 -2021-03-05,60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 -2021-03-05,61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 -2021-03-05,62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 -2021-03-05,63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 -2021-03-05,64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 -2021-03-05,65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 -2021-03-05,66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 -2021-03-05,67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 -2021-03-05,68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 -2021-03-05,69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 -2021-03-05,70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 -2021-03-05,71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 -2021-03-05,72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 -2021-03-05,73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 -2021-03-05,74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 -2021-03-05,75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 -2021-03-05,76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 -2021-03-05,77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 -2021-03-05,78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 -2021-03-05,79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 -2021-03-05,80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 -2021-03-05,81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 -2021-03-05,82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 -2021-03-05,83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 -2021-03-05,84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 -2021-03-05,85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 -2021-03-05,86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 -2021-03-05,87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 -2021-03-05,88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 -2021-03-05,89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 -2021-03-05,90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 -2021-03-05,91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 -2021-03-05,92,Angela,Rogers,ascott2j@goodreads.com,Female,98.119.208.155 -2021-03-05,93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 -2021-03-05,94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 -2021-03-05,95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 -2021-03-05,96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 -2021-03-05,97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 -2021-03-05,98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 -2021-03-05,99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 -2021-03-05,100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 -2021-03-06,101,Michael,Perez,mperez0@chronoengine.com,Male,106.239.70.175 -2021-03-06,102,Shawn,Mccoy,smccoy1@reddit.com,Male,24.165.76.182 -2021-03-06,103,Kathleen,Payne,kpayne2@cargocollective.com,Female,113.207.168.106 -2021-03-06,104,Jimmy,Cooper,jcooper3@cargocollective.com,Male,198.24.63.114 -2021-03-06,105,Katherine,Rice,krice4@typepad.com,Female,36.97.186.238 -2021-03-06,106,Sarah,Ryan,sryan5@gnu.org,Female,119.117.152.40 -2021-03-06,107,Martin,Mcdonald,mmcdonald6@opera.com,Male,8.76.38.115 -2021-03-06,108,Frank,Robinson,frobinson7@wunderground.com,Male,186.14.64.194 -2021-03-06,109,Jennifer,Franklin,jfranklin8@mail.ru,Female,91.216.3.131 -2021-03-06,110,Henry,Welch,hwelch9@list-manage.com,Male,176.35.182.168 -2021-03-06,111,Fred,Snyder,fsnydera@reddit.com,Male,217.106.196.54 -2021-03-06,112,Amy,Dunn,adunnb@nba.com,Female,95.39.163.195 -2021-03-06,113,Kathleen,Meyer,kmeyerc@cdc.gov,Female,164.142.188.214 -2021-03-06,114,Steve,Ferguson,sfergusond@reverbnation.com,Male,138.22.204.251 -2021-03-06,115,Teresa,Hill,thille@dion.ne.jp,Female,82.84.228.235 -2021-03-06,116,Amanda,Harper,aharperf@mail.ru,Female,16.123.56.176 -2021-03-06,117,Kimberly,Ray,krayg@xing.com,Female,48.66.48.12 -2021-03-06,118,Johnny,Knight,jknighth@jalbum.net,Male,99.30.138.123 -2021-03-06,119,Virginia,Freeman,vfreemani@tiny.cc,Female,225.172.182.63 -2021-03-06,120,Anna,Austin,aaustinj@diigo.com,Female,62.111.227.148 -2021-03-06,121,Willie,Hill,whillk@mail.ru,Male,0.86.232.249 -2021-03-06,122,Sean,Harris,sharrisl@zdnet.com,Male,117.165.133.249 -2021-03-06,123,Mildred,Adams,madamsm@usatoday.com,Female,163.44.97.46 -2021-03-06,124,David,Graham,dgrahamn@zimbio.com,Male,78.13.246.202 -2021-03-06,125,Victor,Hunter,vhuntero@ehow.com,Male,64.156.179.139 -2021-03-06,126,Aaron,Ruiz,aruizp@weebly.com,Male,34.194.68.78 -2021-03-06,127,Benjamin,Brooks,bbrooksq@jalbum.net,Male,20.192.189.107 -2021-03-06,128,Lisa,Wilson,lwilsonr@japanpost.jp,Female,199.152.130.217 -2021-03-06,129,Benjamin,King,bkings@comsenz.com,Male,29.189.189.213 -2021-03-06,130,Christina,Williamson,cwilliamsont@boston.com,Female,194.101.52.60 -2021-03-06,131,Jane,Gonzalez,jgonzalezu@networksolutions.com,Female,109.119.12.87 -2021-03-06,132,Thomas,Owens,towensv@psu.edu,Male,84.168.213.153 -2021-03-06,133,Katherine,Moore,kmoorew@naver.com,Female,183.150.65.24 -2021-03-06,134,Jennifer,Stewart,jstewartx@yahoo.com,Female,38.41.244.58 -2021-03-06,135,Sara,Tucker,stuckery@topsy.com,Female,181.130.59.184 -2021-03-06,136,Harold,Ortiz,hortizz@vkontakte.ru,Male,198.231.63.137 -2021-03-06,137,Shirley,James,sjames10@yelp.com,Female,83.27.160.104 -2021-03-06,138,Dennis,Johnson,djohnson11@slate.com,Male,183.178.246.101 -2021-03-06,139,Louise,Weaver,lweaver12@china.com.cn,Female,1.14.110.18 -2021-03-06,140,Maria,Armstrong,marmstrong13@prweb.com,Female,181.142.1.249 -2021-03-06,141,Gloria,Cruz,gcruz14@odnoklassniki.ru,Female,178.232.140.243 -2021-03-06,142,Diana,Spencer,dspencer15@ifeng.com,Female,125.153.138.244 -2021-03-06,143,Kelly,Nguyen,knguyen16@altervista.org,Female,170.13.201.119 -2021-03-06,144,Jane,Rodriguez,jrodriguez17@biblegateway.com,Female,12.102.249.81 -2021-03-06,145,Scott,Brown,sbrown18@geocities.jp,Male,108.174.99.192 -2021-03-06,146,Norma,Cruz,ncruz19@si.edu,Female,201.112.156.197 -2021-03-06,147,Marie,Peters,mpeters1a@mlb.com,Female,231.121.197.144 -2021-03-06,148,Lillian,Carr,lcarr1b@typepad.com,Female,206.179.164.163 -2021-03-06,149,Judy,Nichols,jnichols1c@t-online.de,Female,158.190.209.194 -2021-03-06,150,Billy,Long,blong1d@yahoo.com,Male,175.20.23.160 -2021-03-06,151,Howard,Reid,hreid1e@exblog.jp,Male,118.99.196.20 -2021-03-06,152,Laura,Ferguson,lferguson1f@tuttocitta.it,Female,22.77.87.110 -2021-03-06,153,Anne,Bailey,abailey1g@geocities.com,Female,58.144.159.245 -2021-03-06,154,Rose,Morgan,rmorgan1h@ehow.com,Female,118.127.97.4 -2021-03-06,155,Nicholas,Reyes,nreyes1i@google.ru,Male,50.135.10.252 -2021-03-06,156,Joshua,Kennedy,jkennedy1j@house.gov,Male,154.6.163.209 -2021-03-06,157,Paul,Watkins,pwatkins1k@upenn.edu,Male,177.236.120.87 -2021-03-06,158,Kathryn,Kelly,kkelly1l@businessweek.com,Female,70.28.61.86 -2021-03-06,159,Adam,Armstrong,aarmstrong1m@techcrunch.com,Male,133.235.24.202 -2021-03-06,160,Norma,Wallace,nwallace1n@phoca.cz,Female,241.119.227.128 -2021-03-06,161,Timothy,Reyes,treyes1o@google.cn,Male,86.28.23.26 -2021-03-06,162,Elizabeth,Patterson,epatterson1p@sun.com,Female,139.97.159.149 -2021-03-06,163,Edward,Gomez,egomez1q@google.fr,Male,158.103.108.255 -2021-03-06,164,David,Cox,dcox1r@friendfeed.com,Male,206.80.80.58 -2021-03-06,165,Brenda,Wood,bwood1s@over-blog.com,Female,217.207.44.179 -2021-03-06,166,Adam,Walker,awalker1t@blogs.com,Male,253.211.54.93 -2021-03-06,167,Michael,Hart,mhart1u@wix.com,Male,230.206.200.22 -2021-03-06,168,Jesse,Ellis,jellis1v@google.co.uk,Male,213.254.162.52 -2021-03-06,169,Janet,Powell,jpowell1w@un.org,Female,27.192.194.86 -2021-03-06,170,Helen,Ford,hford1x@creativecommons.org,Female,52.160.102.168 -2021-03-06,171,Gerald,Carpenter,gcarpenter1y@about.me,Male,36.30.194.218 -2021-03-06,172,Kathryn,Oliver,koliver1z@army.mil,Female,202.63.103.69 -2021-03-06,173,Alan,Berry,aberry20@gov.uk,Male,246.157.112.211 -2021-03-06,174,Harry,Andrews,handrews21@ameblo.jp,Male,195.108.0.12 -2021-03-06,175,Andrea,Hall,ahall22@hp.com,Female,149.162.163.28 -2021-03-06,176,Barbara,Wells,bwells23@behance.net,Female,224.70.72.1 -2021-03-06,177,Anne,Wells,awells24@apache.org,Female,180.168.81.153 -2021-03-06,178,Harry,Harper,hharper25@rediff.com,Male,151.87.130.21 -2021-03-06,179,Jack,Ray,jray26@wufoo.com,Male,220.109.38.178 -2021-03-06,180,Phillip,Hamilton,phamilton27@joomla.org,Male,166.40.47.30 -2021-03-06,181,Shirley,Hunter,shunter28@newsvine.com,Female,97.209.140.194 -2021-03-06,182,Arthur,Daniels,adaniels29@reuters.com,Male,5.40.240.86 -2021-03-06,183,Virginia,Rodriguez,vrodriguez2a@walmart.com,Female,96.80.164.184 -2021-03-06,184,Christina,Ryan,cryan2b@hibu.com,Female,56.35.5.52 -2021-03-06,185,Theresa,Mendoza,tmendoza2c@vinaora.com,Female,243.42.0.210 -2021-03-06,186,Jason,Cole,jcole2d@ycombinator.com,Male,198.248.39.129 -2021-03-06,187,Phillip,Bryant,pbryant2e@rediff.com,Male,140.39.116.251 -2021-03-06,188,Adam,Torres,atorres2f@sun.com,Male,101.75.187.135 -2021-03-06,189,Margaret,Johnston,mjohnston2g@ucsd.edu,Female,159.30.69.149 -2021-03-06,190,Paul,Payne,ppayne2h@hhs.gov,Male,199.234.140.220 -2021-03-06,191,Todd,Willis,twillis2i@businessweek.com,Male,191.59.136.214 -2021-03-06,192,Willie,Oliver,woliver2j@noaa.gov,Male,44.212.35.197 -2021-03-06,193,Frances,Robertson,frobertson2k@go.com,Female,31.117.65.136 -2021-03-06,194,Gregory,Hawkins,ghawkins2l@joomla.org,Male,91.3.22.49 -2021-03-06,195,Lisa,Perkins,lperkins2m@si.edu,Female,145.95.31.186 -2021-03-06,196,Jacqueline,Anderson,janderson2n@cargocollective.com,Female,14.176.0.187 -2021-03-06,197,Shirley,Diaz,sdiaz2o@ucla.edu,Female,207.12.95.46 -2021-03-06,198,Nicole,Meyer,nmeyer2p@flickr.com,Female,231.79.115.13 -2021-03-06,199,Mary,Gray,mgray2q@constantcontact.com,Female,210.116.64.253 -2021-03-06,200,Jean,Mcdonald,jmcdonald2r@baidu.com,Female,122.239.235.117 diff --git a/test/integration/001_simple_copy_test/seeds-merge-cols-update/seed.csv b/test/integration/001_simple_copy_test/seeds-merge-cols-update/seed.csv deleted file mode 100644 index 2a2ff674cb0..00000000000 --- a/test/integration/001_simple_copy_test/seeds-merge-cols-update/seed.csv +++ /dev/null @@ -1,204 +0,0 @@ -load_date,id,first_name,last_name,email,gender,ip_address -2021-03-05,1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 -2021-03-05,2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 -2021-03-05,3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 -2021-03-05,4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 -2021-03-05,5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 -2021-03-05,6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 -2021-03-05,7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 -2021-03-05,8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 -2021-03-05,9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 -2021-03-05,10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 -2021-03-05,11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 -2021-03-05,12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 -2021-03-05,13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 -2021-03-05,14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 -2021-03-05,15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 -2021-03-05,16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 -2021-03-05,17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 -2021-03-05,18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 -2021-03-05,19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 -2021-03-05,20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 -2021-03-05,21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 -2021-03-05,22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 -2021-03-05,23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 -2021-03-05,24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 -2021-03-05,25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 -2021-03-05,26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 -2021-03-05,27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 -2021-03-05,28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 -2021-03-05,29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 -2021-03-05,30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 -2021-03-05,31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 -2021-03-05,32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 -2021-03-05,33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 -2021-03-05,34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 -2021-03-05,35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 -2021-03-05,36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 -2021-03-05,37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 -2021-03-05,38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 -2021-03-05,39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 -2021-03-05,40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 -2021-03-05,41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 -2021-03-05,42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 -2021-03-05,43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 -2021-03-05,44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 -2021-03-05,45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 -2021-03-05,46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 -2021-03-05,47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 -2021-03-05,48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 -2021-03-05,49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 -2021-03-05,50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 -2021-03-05,51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 -2021-03-05,52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 -2021-03-05,53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 -2021-03-05,54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 -2021-03-05,55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 -2021-03-05,56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 -2021-03-05,57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 -2021-03-05,58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 -2021-03-05,59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 -2021-03-05,60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 -2021-03-05,61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 -2021-03-05,62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 -2021-03-05,63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 -2021-03-05,64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 -2021-03-05,65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 -2021-03-05,66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 -2021-03-05,67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 -2021-03-05,68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 -2021-03-05,69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 -2021-03-05,70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 -2021-03-05,71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 -2021-03-05,72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 -2021-03-05,73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 -2021-03-05,74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 -2021-03-05,75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 -2021-03-05,76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 -2021-03-05,77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 -2021-03-05,78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 -2021-03-05,79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 -2021-03-05,80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 -2021-03-05,81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 -2021-03-05,82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 -2021-03-05,83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 -2021-03-05,84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 -2021-03-05,85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 -2021-03-05,86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 -2021-03-05,87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 -2021-03-05,88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 -2021-03-05,89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 -2021-03-05,90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 -2021-03-05,91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 -2021-03-05,92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 -2021-03-05,93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 -2021-03-05,94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 -2021-03-05,95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 -2021-03-05,96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 -2021-03-05,97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 -2021-03-05,98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 -2021-03-05,99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 -2021-03-05,100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 -2021-03-06,20,Andrew,Davis,adavisj@reddit.com,Male,9.255.67.25 -2021-03-06,83,Josh,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 -2021-03-06,92,Angela,Scott,ascott2j@goodreads.com,Female,98.119.208.155 -2021-03-06,101,Michael,Perez,mperez0@chronoengine.com,Male,106.239.70.175 -2021-03-06,102,Shawn,Mccoy,smccoy1@reddit.com,Male,24.165.76.182 -2021-03-06,103,Kathleen,Payne,kpayne2@cargocollective.com,Female,113.207.168.106 -2021-03-06,104,Jimmy,Cooper,jcooper3@cargocollective.com,Male,198.24.63.114 -2021-03-06,105,Katherine,Rice,krice4@typepad.com,Female,36.97.186.238 -2021-03-06,106,Sarah,Ryan,sryan5@gnu.org,Female,119.117.152.40 -2021-03-06,107,Martin,Mcdonald,mmcdonald6@opera.com,Male,8.76.38.115 -2021-03-06,108,Frank,Robinson,frobinson7@wunderground.com,Male,186.14.64.194 -2021-03-06,109,Jennifer,Franklin,jfranklin8@mail.ru,Female,91.216.3.131 -2021-03-06,110,Henry,Welch,hwelch9@list-manage.com,Male,176.35.182.168 -2021-03-06,111,Fred,Snyder,fsnydera@reddit.com,Male,217.106.196.54 -2021-03-06,112,Amy,Dunn,adunnb@nba.com,Female,95.39.163.195 -2021-03-06,113,Kathleen,Meyer,kmeyerc@cdc.gov,Female,164.142.188.214 -2021-03-06,114,Steve,Ferguson,sfergusond@reverbnation.com,Male,138.22.204.251 -2021-03-06,115,Teresa,Hill,thille@dion.ne.jp,Female,82.84.228.235 -2021-03-06,116,Amanda,Harper,aharperf@mail.ru,Female,16.123.56.176 -2021-03-06,117,Kimberly,Ray,krayg@xing.com,Female,48.66.48.12 -2021-03-06,118,Johnny,Knight,jknighth@jalbum.net,Male,99.30.138.123 -2021-03-06,119,Virginia,Freeman,vfreemani@tiny.cc,Female,225.172.182.63 -2021-03-06,120,Anna,Austin,aaustinj@diigo.com,Female,62.111.227.148 -2021-03-06,121,Willie,Hill,whillk@mail.ru,Male,0.86.232.249 -2021-03-06,122,Sean,Harris,sharrisl@zdnet.com,Male,117.165.133.249 -2021-03-06,123,Mildred,Adams,madamsm@usatoday.com,Female,163.44.97.46 -2021-03-06,124,David,Graham,dgrahamn@zimbio.com,Male,78.13.246.202 -2021-03-06,125,Victor,Hunter,vhuntero@ehow.com,Male,64.156.179.139 -2021-03-06,126,Aaron,Ruiz,aruizp@weebly.com,Male,34.194.68.78 -2021-03-06,127,Benjamin,Brooks,bbrooksq@jalbum.net,Male,20.192.189.107 -2021-03-06,128,Lisa,Wilson,lwilsonr@japanpost.jp,Female,199.152.130.217 -2021-03-06,129,Benjamin,King,bkings@comsenz.com,Male,29.189.189.213 -2021-03-06,130,Christina,Williamson,cwilliamsont@boston.com,Female,194.101.52.60 -2021-03-06,131,Jane,Gonzalez,jgonzalezu@networksolutions.com,Female,109.119.12.87 -2021-03-06,132,Thomas,Owens,towensv@psu.edu,Male,84.168.213.153 -2021-03-06,133,Katherine,Moore,kmoorew@naver.com,Female,183.150.65.24 -2021-03-06,134,Jennifer,Stewart,jstewartx@yahoo.com,Female,38.41.244.58 -2021-03-06,135,Sara,Tucker,stuckery@topsy.com,Female,181.130.59.184 -2021-03-06,136,Harold,Ortiz,hortizz@vkontakte.ru,Male,198.231.63.137 -2021-03-06,137,Shirley,James,sjames10@yelp.com,Female,83.27.160.104 -2021-03-06,138,Dennis,Johnson,djohnson11@slate.com,Male,183.178.246.101 -2021-03-06,139,Louise,Weaver,lweaver12@china.com.cn,Female,1.14.110.18 -2021-03-06,140,Maria,Armstrong,marmstrong13@prweb.com,Female,181.142.1.249 -2021-03-06,141,Gloria,Cruz,gcruz14@odnoklassniki.ru,Female,178.232.140.243 -2021-03-06,142,Diana,Spencer,dspencer15@ifeng.com,Female,125.153.138.244 -2021-03-06,143,Kelly,Nguyen,knguyen16@altervista.org,Female,170.13.201.119 -2021-03-06,144,Jane,Rodriguez,jrodriguez17@biblegateway.com,Female,12.102.249.81 -2021-03-06,145,Scott,Brown,sbrown18@geocities.jp,Male,108.174.99.192 -2021-03-06,146,Norma,Cruz,ncruz19@si.edu,Female,201.112.156.197 -2021-03-06,147,Marie,Peters,mpeters1a@mlb.com,Female,231.121.197.144 -2021-03-06,148,Lillian,Carr,lcarr1b@typepad.com,Female,206.179.164.163 -2021-03-06,149,Judy,Nichols,jnichols1c@t-online.de,Female,158.190.209.194 -2021-03-06,150,Billy,Long,blong1d@yahoo.com,Male,175.20.23.160 -2021-03-06,151,Howard,Reid,hreid1e@exblog.jp,Male,118.99.196.20 -2021-03-06,152,Laura,Ferguson,lferguson1f@tuttocitta.it,Female,22.77.87.110 -2021-03-06,153,Anne,Bailey,abailey1g@geocities.com,Female,58.144.159.245 -2021-03-06,154,Rose,Morgan,rmorgan1h@ehow.com,Female,118.127.97.4 -2021-03-06,155,Nicholas,Reyes,nreyes1i@google.ru,Male,50.135.10.252 -2021-03-06,156,Joshua,Kennedy,jkennedy1j@house.gov,Male,154.6.163.209 -2021-03-06,157,Paul,Watkins,pwatkins1k@upenn.edu,Male,177.236.120.87 -2021-03-06,158,Kathryn,Kelly,kkelly1l@businessweek.com,Female,70.28.61.86 -2021-03-06,159,Adam,Armstrong,aarmstrong1m@techcrunch.com,Male,133.235.24.202 -2021-03-06,160,Norma,Wallace,nwallace1n@phoca.cz,Female,241.119.227.128 -2021-03-06,161,Timothy,Reyes,treyes1o@google.cn,Male,86.28.23.26 -2021-03-06,162,Elizabeth,Patterson,epatterson1p@sun.com,Female,139.97.159.149 -2021-03-06,163,Edward,Gomez,egomez1q@google.fr,Male,158.103.108.255 -2021-03-06,164,David,Cox,dcox1r@friendfeed.com,Male,206.80.80.58 -2021-03-06,165,Brenda,Wood,bwood1s@over-blog.com,Female,217.207.44.179 -2021-03-06,166,Adam,Walker,awalker1t@blogs.com,Male,253.211.54.93 -2021-03-06,167,Michael,Hart,mhart1u@wix.com,Male,230.206.200.22 -2021-03-06,168,Jesse,Ellis,jellis1v@google.co.uk,Male,213.254.162.52 -2021-03-06,169,Janet,Powell,jpowell1w@un.org,Female,27.192.194.86 -2021-03-06,170,Helen,Ford,hford1x@creativecommons.org,Female,52.160.102.168 -2021-03-06,171,Gerald,Carpenter,gcarpenter1y@about.me,Male,36.30.194.218 -2021-03-06,172,Kathryn,Oliver,koliver1z@army.mil,Female,202.63.103.69 -2021-03-06,173,Alan,Berry,aberry20@gov.uk,Male,246.157.112.211 -2021-03-06,174,Harry,Andrews,handrews21@ameblo.jp,Male,195.108.0.12 -2021-03-06,175,Andrea,Hall,ahall22@hp.com,Female,149.162.163.28 -2021-03-06,176,Barbara,Wells,bwells23@behance.net,Female,224.70.72.1 -2021-03-06,177,Anne,Wells,awells24@apache.org,Female,180.168.81.153 -2021-03-06,178,Harry,Harper,hharper25@rediff.com,Male,151.87.130.21 -2021-03-06,179,Jack,Ray,jray26@wufoo.com,Male,220.109.38.178 -2021-03-06,180,Phillip,Hamilton,phamilton27@joomla.org,Male,166.40.47.30 -2021-03-06,181,Shirley,Hunter,shunter28@newsvine.com,Female,97.209.140.194 -2021-03-06,182,Arthur,Daniels,adaniels29@reuters.com,Male,5.40.240.86 -2021-03-06,183,Virginia,Rodriguez,vrodriguez2a@walmart.com,Female,96.80.164.184 -2021-03-06,184,Christina,Ryan,cryan2b@hibu.com,Female,56.35.5.52 -2021-03-06,185,Theresa,Mendoza,tmendoza2c@vinaora.com,Female,243.42.0.210 -2021-03-06,186,Jason,Cole,jcole2d@ycombinator.com,Male,198.248.39.129 -2021-03-06,187,Phillip,Bryant,pbryant2e@rediff.com,Male,140.39.116.251 -2021-03-06,188,Adam,Torres,atorres2f@sun.com,Male,101.75.187.135 -2021-03-06,189,Margaret,Johnston,mjohnston2g@ucsd.edu,Female,159.30.69.149 -2021-03-06,190,Paul,Payne,ppayne2h@hhs.gov,Male,199.234.140.220 -2021-03-06,191,Todd,Willis,twillis2i@businessweek.com,Male,191.59.136.214 -2021-03-06,192,Willie,Oliver,woliver2j@noaa.gov,Male,44.212.35.197 -2021-03-06,193,Frances,Robertson,frobertson2k@go.com,Female,31.117.65.136 -2021-03-06,194,Gregory,Hawkins,ghawkins2l@joomla.org,Male,91.3.22.49 -2021-03-06,195,Lisa,Perkins,lperkins2m@si.edu,Female,145.95.31.186 -2021-03-06,196,Jacqueline,Anderson,janderson2n@cargocollective.com,Female,14.176.0.187 -2021-03-06,197,Shirley,Diaz,sdiaz2o@ucla.edu,Female,207.12.95.46 -2021-03-06,198,Nicole,Meyer,nmeyer2p@flickr.com,Female,231.79.115.13 -2021-03-06,199,Mary,Gray,mgray2q@constantcontact.com,Female,210.116.64.253 -2021-03-06,200,Jean,Mcdonald,jmcdonald2r@baidu.com,Female,122.239.235.117 diff --git a/test/integration/001_simple_copy_test/test_simple_copy.py b/test/integration/001_simple_copy_test/test_simple_copy.py index c0a00230310..3ccc0d5305f 100644 --- a/test/integration/001_simple_copy_test/test_simple_copy.py +++ b/test/integration/001_simple_copy_test/test_simple_copy.py @@ -105,262 +105,6 @@ def test__presto__simple_copy(self): self.assertManyTablesEqual(["seed", "view_model", "materialized"]) - @use_profile("snowflake") - def test__snowflake__simple_copy(self): - self.use_default_project({ - "data-paths": [self.dir("seed-initial")], - "seeds": { - 'quote_columns': False, - } - }) - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - results = self.run_dbt() - self.assertEqual(len(results), 7) - - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"]) - - self.use_default_project({"data-paths": [self.dir("seed-update")]}) - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - results = self.run_dbt() - self.assertEqual(len(results), 7) - - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"]) - - self.use_default_project({ - "test-paths": [self.dir("tests")], - "data-paths": [self.dir("seed-update")], - }) - self.run_dbt(['test']) - - @use_profile("snowflake") - def test__snowflake__simple_copy__quoting_off(self): - self.use_default_project({ - "quoting": {"identifier": False}, - "data-paths": [self.dir("snowflake-seed-initial")], - }) - - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - results = self.run_dbt() - self.assertEqual(len(results), 7) - - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"]) - - self.use_default_project({ - "data-paths": [self.dir("snowflake-seed-update")], - "quoting": {"identifier": False}, - }) - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - results = self.run_dbt() - self.assertEqual(len(results), 7) - - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"]) - - self.use_default_project({ - "test-paths": [self.dir("tests")], - "data-paths": [self.dir("snowflake-seed-update")], - "quoting": {"identifier": False}, - }) - self.run_dbt(['test']) - - @use_profile("snowflake") - def test__snowflake__seed__quoting_switch(self): - self.use_default_project({ - "quoting": {"identifier": False}, - "data-paths": [self.dir("snowflake-seed-initial")], - }) - - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - - self.use_default_project({ - "data-paths": [self.dir("snowflake-seed-update")], - "quoting": {"identifier": True}, - }) - results = self.run_dbt(["seed"], expect_pass=False) - - self.use_default_project({ - "test-paths": [self.dir("tests")], - "data-paths": [self.dir("snowflake-seed-initial")], - }) - self.run_dbt(['test']) - - @use_profile("bigquery") - def test__bigquery__simple_copy(self): - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - results = self.run_dbt() - self.assertEqual(len(results), 7) - - self.assertTablesEqual("seed", "view_model") - self.assertTablesEqual("seed", "incremental") - self.assertTablesEqual("seed", "materialized") - self.assertTablesEqual("seed", "get_and_ref") - - self.use_default_project({"data-paths": [self.dir("seed-update")]}) - - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - results = self.run_dbt() - self.assertEqual(len(results), 7) - - self.assertTablesEqual("seed", "view_model") - self.assertTablesEqual("seed", "incremental") - self.assertTablesEqual("seed", "materialized") - self.assertTablesEqual("seed", "get_and_ref") - - -class TestSimpleCopyQuotingIdentifierOn(BaseTestSimpleCopy): - @property - def project_config(self): - return self.seed_quote_cfg_with({ - 'quoting': { - 'identifier': True, - }, - }) - - @use_profile("snowflake") - def test__snowflake__simple_copy__quoting_on(self): - self.use_default_project({ - "data-paths": [self.dir("snowflake-seed-initial")], - }) - - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - results = self.run_dbt() - self.assertEqual(len(results), 7) - - self.assertManyTablesEqual(["seed", "view_model", "incremental", "materialized", "get_and_ref"]) - - self.use_default_project({ - "data-paths": [self.dir("snowflake-seed-update")], - }) - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - results = self.run_dbt() - self.assertEqual(len(results), 7) - - self.assertManyTablesEqual(["seed", "view_model", "incremental", "materialized", "get_and_ref"]) - - # can't run the test as this one's identifiers will be the wrong case - - -class BaseLowercasedSchemaTest(BaseTestSimpleCopy): - def unique_schema(self): - # bypass the forced uppercasing that unique_schema() does on snowflake - return super().unique_schema().lower() - - -class TestSnowflakeSimpleLowercasedSchemaCopy(BaseLowercasedSchemaTest): - @use_profile('snowflake') - def test__snowflake__simple_copy(self): - self.use_default_project({"data-paths": [self.dir("snowflake-seed-initial")]}) - - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - results = self.run_dbt() - self.assertEqual(len(results), 7) - - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"]) - - self.use_default_project({"data-paths": [self.dir("snowflake-seed-update")]}) - - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - results = self.run_dbt() - self.assertEqual(len(results), 7) - - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"]) - - self.use_default_project({ - "test-paths": [self.dir("tests")], - "data-paths": [self.dir("snowflake-seed-update")], - }) - self.run_dbt(['test']) - - -class TestSnowflakeSimpleLowercasedSchemaQuoted(BaseLowercasedSchemaTest): - @property - def project_config(self): - return self.seed_quote_cfg_with({ - 'quoting': {'identifier': False, 'schema': True} - }) - - @use_profile("snowflake") - def test__snowflake__seed__quoting_switch_schema_lower(self): - self.use_default_project({ - "data-paths": [self.dir("snowflake-seed-initial")], - }) - - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - # this is intentional - should not error! - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - - self.use_default_project({ - "data-paths": [self.dir("snowflake-seed-update")], - "quoting": {"identifier": False, "schema": False}, - }) - results = self.run_dbt(["seed"], expect_pass=False) - - -class TestSnowflakeSimpleUppercasedSchemaQuoted(BaseTestSimpleCopy): - @property - def project_config(self): - return self.seed_quote_cfg_with({ - 'quoting': {'identifier': False, 'schema': True} - }) - - @use_profile("snowflake") - def test__snowflake__seed__quoting_switch_schema_upper(self): - self.use_default_project({ - "data-paths": [self.dir("snowflake-seed-initial")], - }) - - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - # this is intentional - should not error! - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - - self.use_default_project({ - "data-paths": [self.dir("snowflake-seed-update")], - "quoting": {"identifier": False, "schema": False}, - }) - results = self.run_dbt(["seed"]) - - -class TestSnowflakeIncrementalOverwrite(BaseTestSimpleCopy): - @property - def models(self): - return self.dir("models-snowflake") - - @use_profile("snowflake") - def test__snowflake__incremental_overwrite(self): - self.use_default_project({ - "data-paths": [self.dir("snowflake-seed-initial")], - }) - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 1) - - results = self.run_dbt(["run"], expect_pass=False) - self.assertEqual(len(results), 1) - - # Setting the incremental_strategy should make this succeed - self.use_default_project({ - "models": { - "incremental_strategy": "delete+insert" - }, - "data-paths": [self.dir("snowflake-seed-update")], - }) - - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 1) - class TestShouting(BaseTestSimpleCopy): @property @@ -468,51 +212,3 @@ def test_postgres_no_create_schemas(self): 'create schema if not exists' in msg, f'did not expect schema creation: {msg}' ) - - -class TestIncrementalMergeColumns(BaseTestSimpleCopy): - @property - def models(self): - return self.dir("models-merge-update") - - @property - def project_config(self): - return { - "seeds": { - "quote_columns": False - } - } - - def seed_and_run(self): - self.run_dbt(["seed"]) - self.run_dbt(["run"]) - - @use_profile("bigquery") - def test__bigquery__incremental_merge_columns(self): - self.use_default_project({ - "data-paths": ["seeds-merge-cols-initial"] - }) - self.seed_and_run() - self.use_default_project({ - "data-paths": ["seeds-merge-cols-update"] - }) - self.seed_and_run() - self.assertTablesEqual("incremental_update_cols", "expected_result") - - @use_profile("snowflake") - def test__snowflake__incremental_merge_columns(self): - self.use_default_project({ - "data-paths": ["seeds-merge-cols-initial"], - "seeds": { - "quote_columns": False - } - }) - self.seed_and_run() - self.use_default_project({ - "data-paths": ["seeds-merge-cols-update"], - "seeds": { - "quote_columns": False - } - }) - self.seed_and_run() - self.assertTablesEqual("incremental_update_cols", "expected_result") diff --git a/test/integration/002_varchar_widening_test/test_varchar_widening.py b/test/integration/002_varchar_widening_test/test_varchar_widening.py index 1485766122c..2ee5cdd5b0f 100644 --- a/test/integration/002_varchar_widening_test/test_varchar_widening.py +++ b/test/integration/002_varchar_widening_test/test_varchar_widening.py @@ -27,19 +27,3 @@ def test__postgres__varchar_widening(self): self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") - - @use_profile('snowflake') - def test__snowflake__varchar_widening(self): - self.run_sql_file("seed.sql") - - results = self.run_dbt() - self.assertEqual(len(results), 2) - - self.assertManyTablesEqual(["SEED", "INCREMENTAL", "MATERIALIZED"]) - - self.run_sql_file("update.sql") - - results = self.run_dbt() - self.assertEqual(len(results), 2) - - self.assertManyTablesEqual(["SEED", "INCREMENTAL", "MATERIALIZED"]) diff --git a/test/integration/003_simple_reference_test/test_simple_reference.py b/test/integration/003_simple_reference_test/test_simple_reference.py index 9db7cc5d1ba..88dc484275c 100644 --- a/test/integration/003_simple_reference_test/test_simple_reference.py +++ b/test/integration/003_simple_reference_test/test_simple_reference.py @@ -66,28 +66,6 @@ def test__postgres__simple_reference(self): self.assertTablesEqual("summary_expected","ephemeral_summary") self.assertTablesEqual("summary_expected","view_using_ref") - @use_profile('snowflake') - def test__snowflake__simple_reference(self): - - results = self.run_dbt() - self.assertEqual(len(results), 8) - - # Copies should match - self.assertManyTablesEqual( - ["SEED", "INCREMENTAL_COPY", "MATERIALIZED_COPY", "VIEW_COPY"], - ["SUMMARY_EXPECTED", "INCREMENTAL_SUMMARY", "MATERIALIZED_SUMMARY", "VIEW_SUMMARY", "EPHEMERAL_SUMMARY"] - ) - - self.run_sql_file("update.sql") - - results = self.run_dbt() - self.assertEqual(len(results), 8) - - self.assertManyTablesEqual( - ["SEED", "INCREMENTAL_COPY", "MATERIALIZED_COPY", "VIEW_COPY"], - ["SUMMARY_EXPECTED", "INCREMENTAL_SUMMARY", "MATERIALIZED_SUMMARY", "VIEW_SUMMARY", "EPHEMERAL_SUMMARY"] - ) - @use_profile('postgres') def test__postgres__simple_reference_with_models(self): @@ -140,57 +118,6 @@ def test__postgres__simple_reference_with_models_and_children(self): self.assertTrue('ephemeral_summary' in created_models) self.assertEqual(created_models['ephemeral_summary'], 'table') - @use_profile('snowflake') - def test__snowflake__simple_reference_with_models(self): - - # Run materialized_copy & ephemeral_copy - # ephemeral_copy should not actually be materialized b/c it is ephemeral - results = self.run_dbt( - ['run', '--models', 'materialized_copy', 'ephemeral_copy'] - ) - self.assertEqual(len(results), 1) - - # Copies should match - self.assertTablesEqual("SEED", "MATERIALIZED_COPY") - - created_models = self.get_models_in_schema() - self.assertTrue('MATERIALIZED_COPY' in created_models) - - @use_profile('snowflake') - def test__snowflake__simple_reference_with_models_and_children(self): - - # Run materialized_copy, ephemeral_copy, and their dependents - # ephemeral_copy should not actually be materialized b/c it is ephemeral - # the dependent ephemeral_summary, however, should be materialized as a table - results = self.run_dbt( - ['run', '--models', 'materialized_copy+', 'ephemeral_copy+'] - ) - self.assertEqual(len(results), 3) - - # Copies should match - self.assertManyTablesEqual( - ["SEED", "MATERIALIZED_COPY"], - ["SUMMARY_EXPECTED", "MATERIALIZED_SUMMARY", "EPHEMERAL_SUMMARY"] - ) - - created_models = self.get_models_in_schema() - - self.assertFalse('INCREMENTAL_COPY' in created_models) - self.assertFalse('INCREMENTAL_SUMMARY' in created_models) - self.assertFalse('VIEW_COPY' in created_models) - self.assertFalse('VIEW_SUMMARY' in created_models) - - # make sure this wasn't errantly materialized - self.assertFalse('EPHEMERAL_COPY' in created_models) - - self.assertTrue('MATERIALIZED_COPY' in created_models) - self.assertTrue('MATERIALIZED_SUMMARY' in created_models) - self.assertEqual(created_models['MATERIALIZED_COPY'], 'table') - self.assertEqual(created_models['MATERIALIZED_SUMMARY'], 'table') - - self.assertTrue('EPHEMERAL_SUMMARY' in created_models) - self.assertEqual(created_models['EPHEMERAL_SUMMARY'], 'table') - class TestErrorReference(DBTIntegrationTest): @property diff --git a/test/integration/004_simple_snapshot_test/invalidate_bigquery.sql b/test/integration/004_simple_snapshot_test/invalidate_bigquery.sql deleted file mode 100644 index d4641d45121..00000000000 --- a/test/integration/004_simple_snapshot_test/invalidate_bigquery.sql +++ /dev/null @@ -1,12 +0,0 @@ - --- update records 11 - 21. Change email and updated_at field -update {database}.{schema}.seed set - updated_at = timestamp_add(updated_at, interval 1 hour), - email = case when id = 20 then 'pfoxj@creativecommons.org' else concat('new_', email) end -where id >= 10 and id <= 20; - - --- invalidate records 11 - 21 -update {database}.{schema}.snapshot_expected set - dbt_valid_to = timestamp_add(updated_at, interval 1 hour) -where id >= 10 and id <= 20; diff --git a/test/integration/004_simple_snapshot_test/invalidate_snowflake.sql b/test/integration/004_simple_snapshot_test/invalidate_snowflake.sql deleted file mode 100644 index 57c4b71d635..00000000000 --- a/test/integration/004_simple_snapshot_test/invalidate_snowflake.sql +++ /dev/null @@ -1,12 +0,0 @@ - --- update records 11 - 21. Change email and updated_at field -update {database}.{schema}.seed set - updated_at = DATEADD(hour, 1, updated_at), - email = case when id = 20 then 'pfoxj@creativecommons.org' else 'new_' || email end -where id >= 10 and id <= 20; - - --- invalidate records 11 - 21 -update {database}.{schema}.snapshot_expected set - dbt_valid_to = DATEADD(hour, 1, updated_at) -where id >= 10 and id <= 20; diff --git a/test/integration/004_simple_snapshot_test/macros/test_no_overlaps.sql b/test/integration/004_simple_snapshot_test/macros/test_no_overlaps.sql index 6d432193c27..9bdfdd264da 100644 --- a/test/integration/004_simple_snapshot_test/macros/test_no_overlaps.sql +++ b/test/integration/004_simple_snapshot_test/macros/test_no_overlaps.sql @@ -6,11 +6,6 @@ {% do return("id || '-' || first_name") %} {%- endmacro %} - -{% macro bigquery__get_snapshot_unique_id() -%} - {%- do return('concat(cast(id as string), "-", first_name)') -%} -{%- endmacro %} - {# mostly copy+pasted from dbt_utils, but I removed some parameters and added a query that calls get_snapshot_unique_id diff --git a/test/integration/004_simple_snapshot_test/seed_bq.sql b/test/integration/004_simple_snapshot_test/seed_bq.sql deleted file mode 100644 index 5ea93fee47b..00000000000 --- a/test/integration/004_simple_snapshot_test/seed_bq.sql +++ /dev/null @@ -1,81 +0,0 @@ -create table {database}.{schema}.seed ( - id INT64, - first_name STRING, - last_name STRING, - email STRING, - gender STRING, - ip_address STRING, - updated_at TIMESTAMP -); - -create table {database}.{schema}.snapshot_expected ( - id INT64, - first_name STRING, - last_name STRING, - email STRING, - gender STRING, - ip_address STRING, - - -- snapshotting fields - updated_at TIMESTAMP, - dbt_valid_from TIMESTAMP, - dbt_valid_to TIMESTAMP, - dbt_scd_id STRING, - dbt_updated_at TIMESTAMP -); - - --- seed inserts -insert {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values -(1, 'Judith', 'Kennedy', 'jkennedy0@phpbb.com', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), -(2, 'Arthur', 'Kelly', 'akelly1@eepurl.com', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), -(3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'), -(4, 'Ralph', 'Turner', 'rturner3@hp.com', 'Male', '157.83.76.114', '2016-08-08 00:06:51'), -(5, 'Laura', 'Gonzales', 'lgonzales4@howstuffworks.com', 'Female', '30.54.105.168', '2016-09-01 08:25:38'), -(6, 'Katherine', 'Lopez', 'klopez5@yahoo.co.jp', 'Female', '169.138.46.89', '2016-08-30 18:52:11'), -(7, 'Jeremy', 'Hamilton', 'jhamilton6@mozilla.org', 'Male', '231.189.13.133', '2016-07-17 02:09:46'), -(8, 'Heather', 'Rose', 'hrose7@goodreads.com', 'Female', '87.165.201.65', '2015-12-29 22:03:56'), -(9, 'Gregory', 'Kelly', 'gkelly8@trellian.com', 'Male', '154.209.99.7', '2016-03-24 21:18:16'), -(10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '2016-08-20 15:44:49'), -(11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '2016-02-27 01:41:48'), -(12, 'Russell', 'Lawrence', 'rlawrenceb@qq.com', 'Male', '189.115.73.4', '2016-06-11 03:07:09'), -(13, 'Michelle', 'Montgomery', 'mmontgomeryc@scientificamerican.com', 'Female', '243.220.95.82', '2016-06-18 16:27:19'), -(14, 'Walter', 'Castillo', 'wcastillod@pagesperso-orange.fr', 'Male', '71.159.238.196', '2016-10-06 01:55:44'), -(15, 'Robin', 'Mills', 'rmillse@vkontakte.ru', 'Female', '172.190.5.50', '2016-10-31 11:41:21'), -(16, 'Raymond', 'Holmes', 'rholmesf@usgs.gov', 'Male', '148.153.166.95', '2016-10-03 08:16:38'), -(17, 'Gary', 'Bishop', 'gbishopg@plala.or.jp', 'Male', '161.108.182.13', '2016-08-29 19:35:20'), -(18, 'Anna', 'Riley', 'arileyh@nasa.gov', 'Female', '253.31.108.22', '2015-12-11 04:34:27'), -(19, 'Sarah', 'Knight', 'sknighti@foxnews.com', 'Female', '222.220.3.177', '2016-09-26 00:49:06'), -(20, 'Phyllis', 'Fox', null, 'Female', '163.191.232.95', '2016-08-21 10:35:19'); - - --- populate snapshot table -insert {database}.{schema}.snapshot_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - cast(null as timestamp) as dbt_valid_to, - updated_at as dbt_updated_at, - to_hex(md5(concat(cast(id as string), '-', first_name, '|', cast(updated_at as string)))) as dbt_scd_id -from {database}.{schema}.seed; - diff --git a/test/integration/004_simple_snapshot_test/test-check-col-snapshots-bq/snapshot.sql b/test/integration/004_simple_snapshot_test/test-check-col-snapshots-bq/snapshot.sql deleted file mode 100644 index 9c84597561e..00000000000 --- a/test/integration/004_simple_snapshot_test/test-check-col-snapshots-bq/snapshot.sql +++ /dev/null @@ -1,29 +0,0 @@ -{% snapshot snapshot_actual %} - {# this used to be check_cols=('email',), which ought to be totally valid, - but is not because type systems are hard. #} - {{ - config( - target_project=var('target_database', database), - target_dataset=var('target_schema', schema), - unique_key='concat(cast(id as string) , "-", first_name)', - strategy='check', - check_cols=['email'], - ) - }} - select * from `{{target.database}}`.`{{schema}}`.seed -{% endsnapshot %} - - -{# This should be exactly the same #} -{% snapshot snapshot_checkall %} - {{ - config( - target_project=var('target_database', database), - target_dataset=var('target_schema', schema), - unique_key='concat(cast(id as string) , "-", first_name)', - strategy='check', - check_cols='all', - ) - }} - select * from `{{target.database}}`.`{{schema}}`.seed -{% endsnapshot %} diff --git a/test/integration/004_simple_snapshot_test/test-snapshots-bq/snapshot.sql b/test/integration/004_simple_snapshot_test/test-snapshots-bq/snapshot.sql deleted file mode 100644 index 7ffdedbcc2b..00000000000 --- a/test/integration/004_simple_snapshot_test/test-snapshots-bq/snapshot.sql +++ /dev/null @@ -1,19 +0,0 @@ -{% snapshot snapshot_actual %} - - {{ - config( - target_project=var('target_database', database), - target_dataset=var('target_schema', schema), - unique_key='concat(cast(id as string) , "-", first_name)', - strategy='timestamp', - updated_at='updated_at', - ) - }} - - {% if var('invalidate_hard_deletes', 'false') | as_bool %} - {{ config(invalidate_hard_deletes=True) }} - {% endif %} - - select * from `{{target.database}}`.`{{schema}}`.seed - -{% endsnapshot %} diff --git a/test/integration/004_simple_snapshot_test/test_simple_snapshot.py b/test/integration/004_simple_snapshot_test/test_simple_snapshot.py index e2e4f30a3c4..a5f51be3bdd 100644 --- a/test/integration/004_simple_snapshot_test/test_simple_snapshot.py +++ b/test/integration/004_simple_snapshot_test/test_simple_snapshot.py @@ -28,10 +28,7 @@ def dbt_run_seed_snapshot(self): self.assertEqual(len(results), self.NUM_SNAPSHOT_MODELS) def assert_case_tables_equal(self, actual, expected): - if self.adapter_type == 'snowflake': - actual = actual.upper() - expected = expected.upper() - + # this does something different on snowflake, but here it's just assertTablesEqual self.assertTablesEqual(actual, expected) def assert_expected(self): @@ -69,34 +66,6 @@ def test__postgres__simple_snapshot(self): self.assert_expected() - @use_profile('snowflake') - def test__snowflake__simple_snapshot(self): - self.dbt_run_seed_snapshot() - - self.assert_expected() - - self.run_sql_file("invalidate_snowflake.sql") - self.run_sql_file("update.sql") - - results = self.run_snapshot() - self.assertEqual(len(results), self.NUM_SNAPSHOT_MODELS) - - self.assert_expected() - - @use_profile('redshift') - def test__redshift__simple_snapshot(self): - self.dbt_run_seed_snapshot() - - self.assert_expected() - - self.run_sql_file("invalidate_postgres.sql") - self.run_sql_file("update.sql") - - results = self.run_snapshot() - self.assertEqual(len(results), self.NUM_SNAPSHOT_MODELS) - - self.assert_expected() - class TestSimpleColumnSnapshotFiles(DBTIntegrationTest): @@ -124,8 +93,6 @@ def _run_snapshot_test(self): self.run_dbt(['seed']) self.run_dbt(['snapshot']) database = self.default_database - if self.adapter_type == 'bigquery': - database = self.adapter.quote(database) results = self.run_sql( 'select * from {}.{}.my_snapshot'.format(database, self.unique_schema()), fetch='all' @@ -159,18 +126,6 @@ def _run_snapshot_test(self): def test_postgres_renamed_source(self): self._run_snapshot_test() - @use_profile('snowflake') - def test_snowflake_renamed_source(self): - self._run_snapshot_test() - - @use_profile('redshift') - def test_redshift_renamed_source(self): - self._run_snapshot_test() - - @use_profile('bigquery') - def test_bigquery_renamed_source(self): - self._run_snapshot_test() - class TestCustomSnapshotFiles(BaseSimpleSnapshotTest): @property @@ -326,145 +281,6 @@ def project_config(self): } -class TestSimpleSnapshotFilesBigquery(DBTIntegrationTest): - @property - def schema(self): - return "simple_snapshot_004" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - "snapshot-paths": ['test-snapshots-bq'], - 'macro-paths': ['macros'], - } - - def assert_expected(self): - self.run_dbt(['test']) - self.assertTablesEqual('snapshot_actual', 'snapshot_expected') - - @use_profile('bigquery') - def test__bigquery__simple_snapshot(self): - self.run_sql_file("seed_bq.sql") - - self.run_dbt(["snapshot"]) - - self.assert_expected() - - self.run_sql_file("invalidate_bigquery.sql") - self.run_sql_file("update_bq.sql") - - self.run_dbt(["snapshot"]) - - self.assert_expected() - - @use_profile('bigquery') - def test__bigquery__snapshot_with_new_field(self): - - self.run_sql_file("seed_bq.sql") - - self.run_dbt(["snapshot"]) - - self.assertTablesEqual("snapshot_expected", "snapshot_actual") - - self.run_sql_file("invalidate_bigquery.sql") - self.run_sql_file("update_bq.sql") - - # This adds new fields to the source table, and updates the expected snapshot output accordingly - self.run_sql_file("add_column_to_source_bq.sql") - - self.run_dbt(["snapshot"]) - - # A more thorough test would assert that snapshotted == expected, but BigQuery does not support the - # "EXCEPT DISTINCT" operator on nested fields! Instead, just check that schemas are congruent. - - expected_cols = self.get_table_columns( - database=self.default_database, - schema=self.unique_schema(), - table='snapshot_expected' - ) - snapshotted_cols = self.get_table_columns( - database=self.default_database, - schema=self.unique_schema(), - table='snapshot_actual' - ) - - self.assertTrue(len(expected_cols) > 0, "source table does not exist -- bad test") - self.assertEqual(len(expected_cols), len(snapshotted_cols), "actual and expected column lengths are different") - - for (expected_col, actual_col) in zip(expected_cols, snapshotted_cols): - expected_name, expected_type, _ = expected_col - actual_name, actual_type, _ = actual_col - self.assertTrue(expected_name is not None) - self.assertTrue(expected_type is not None) - - self.assertEqual(expected_name, actual_name, "names are different") - self.assertEqual(expected_type, actual_type, "data types are different") - - -class TestCrossDBSnapshotFiles(DBTIntegrationTest): - setup_alternate_db = True - - @property - def schema(self): - return "simple_snapshot_004" - - @property - def models(self): - return "models" - - @property - def project_config(self): - if self.adapter_type == 'snowflake': - paths = ['test-snapshots-pg'] - else: - paths = ['test-snapshots-bq'] - return { - 'config-version': 2, - 'snapshot-paths': paths, - 'macro-paths': ['macros'], - } - - def run_snapshot(self): - return self.run_dbt(['snapshot', '--vars', '{{"target_database": {}}}'.format(self.alternative_database)]) - - @use_profile('snowflake') - def test__snowflake__cross_snapshot(self): - self.run_sql_file("seed.sql") - - results = self.run_snapshot() - self.assertEqual(len(results), 1) - - self.assertTablesEqual("SNAPSHOT_EXPECTED", "SNAPSHOT_ACTUAL", table_b_db=self.alternative_database) - - self.run_sql_file("invalidate_snowflake.sql") - self.run_sql_file("update.sql") - - results = self.run_snapshot() - self.assertEqual(len(results), 1) - - self.assertTablesEqual("SNAPSHOT_EXPECTED", "SNAPSHOT_ACTUAL", table_b_db=self.alternative_database) - - @use_profile('bigquery') - def test__bigquery__cross_snapshot(self): - self.run_sql_file("seed_bq.sql") - - self.run_snapshot() - - self.assertTablesEqual("snapshot_expected", "snapshot_actual", table_b_db=self.alternative_database) - - self.run_sql_file("invalidate_bigquery.sql") - self.run_sql_file("update_bq.sql") - - self.run_snapshot() - - self.assertTablesEqual("snapshot_expected", "snapshot_actual", table_b_db=self.alternative_database) - - class TestCrossSchemaSnapshotFiles(DBTIntegrationTest): NUM_SNAPSHOT_MODELS = 1 @@ -624,81 +440,6 @@ def project_config(self): } -class TestCheckColsBigquery(TestSimpleSnapshotFilesBigquery): - def _assertTablesEqualSql(self, relation_a, relation_b, columns=None): - # When building the equality tests, only test columns that don't start - # with 'dbt_', because those are time-sensitive - if columns is None: - columns = [c for c in self.get_relation_columns(relation_a) if not c[0].lower().startswith('dbt_')] - return super()._assertTablesEqualSql(relation_a, relation_b, columns=columns) - - def assert_expected(self): - super().assert_expected() - self.assertTablesEqual('snapshot_checkall', 'snapshot_expected') - - @property - def project_config(self): - return { - 'config-version': 2, - "data-paths": ['data'], - "snapshot-paths": ['test-check-col-snapshots-bq'], - 'macro-paths': ['macros'], - } - - @use_profile('bigquery') - def test__bigquery__snapshot_with_new_field(self): - self.use_default_project() - self.use_profile('bigquery') - - self.run_sql_file("seed_bq.sql") - - self.run_dbt(["snapshot"]) - - self.assertTablesEqual("snapshot_expected", "snapshot_actual") - self.assertTablesEqual("snapshot_expected", "snapshot_checkall") - - self.run_sql_file("invalidate_bigquery.sql") - self.run_sql_file("update_bq.sql") - - # This adds new fields to the source table, and updates the expected snapshot output accordingly - self.run_sql_file("add_column_to_source_bq.sql") - - # check_cols='all' will replace the changed field - self.run_dbt(['snapshot']) - - # A more thorough test would assert that snapshotted == expected, but BigQuery does not support the - # "EXCEPT DISTINCT" operator on nested fields! Instead, just check that schemas are congruent. - - expected_cols = self.get_table_columns( - database=self.default_database, - schema=self.unique_schema(), - table='snapshot_expected' - ) - snapshotted_cols = self.get_table_columns( - database=self.default_database, - schema=self.unique_schema(), - table='snapshot_actual' - ) - snapshotted_all_cols = self.get_table_columns( - database=self.default_database, - schema=self.unique_schema(), - table='snapshot_checkall' - ) - - self.assertTrue(len(expected_cols) > 0, "source table does not exist -- bad test") - self.assertEqual(len(expected_cols), len(snapshotted_cols), "actual and expected column lengths are different") - self.assertEqual(len(expected_cols), len(snapshotted_all_cols)) - - for (expected_col, actual_col) in zip(expected_cols, snapshotted_cols): - expected_name, expected_type, _ = expected_col - actual_name, actual_type, _ = actual_col - self.assertTrue(expected_name is not None) - self.assertTrue(expected_type is not None) - - self.assertEqual(expected_name, actual_name, "names are different") - self.assertEqual(expected_type, actual_type, "data types are different") - - class TestLongText(DBTIntegrationTest): @property @@ -820,10 +561,7 @@ def models(self): @property def project_config(self): - if self.adapter_type == 'bigquery': - paths = ['test-snapshots-bq'] - else: - paths = ['test-snapshots-pg'] + paths = ['test-snapshots-pg'] return { 'config-version': 2, @@ -837,28 +575,10 @@ def test__postgres__snapshot_hard_delete(self): self.run_sql_file('seed_pg.sql') self._test_snapshot_hard_delete() - @use_profile('bigquery') - def test__bigquery__snapshot_hard_delete(self): - self.run_sql_file('seed_bq.sql') - self._test_snapshot_hard_delete() - - @use_profile('snowflake') - def test__snowflake__snapshot_hard_delete(self): - self.run_sql_file('seed.sql') - self._test_snapshot_hard_delete() - - @use_profile('redshift') - def test__redshift__snapshot_hard_delete(self): - self.run_sql_file('seed.sql') - self._test_snapshot_hard_delete() - def _test_snapshot_hard_delete(self): self._snapshot() - if self.adapter_type == 'snowflake': - self.assertTablesEqual("SNAPSHOT_EXPECTED", "SNAPSHOT_ACTUAL") - else: - self.assertTablesEqual("snapshot_expected", "snapshot_actual") + self.assertTablesEqual("snapshot_expected", "snapshot_actual") self._invalidated_snapshot_datetime = None self._revived_snapshot_datetime = None @@ -877,8 +597,6 @@ def _snapshot(self): def _delete_records(self): database = self.default_database - if self.adapter_type == 'bigquery': - database = self.adapter.quote(database) self.run_sql( 'delete from {}.{}.seed where id >= 10;'.format(database, self.unique_schema()) @@ -888,8 +606,6 @@ def _snapshot_and_assert_invalidated(self): self._invalidated_snapshot_datetime = self._snapshot() database = self.default_database - if self.adapter_type == 'bigquery': - database = self.adapter.quote(database) snapshotted = self.run_sql( ''' @@ -910,8 +626,6 @@ def _snapshot_and_assert_invalidated(self): def _revive_records(self): database = self.default_database - if self.adapter_type == 'bigquery': - database = self.adapter.quote(database) revival_timestamp = datetime.now(pytz.UTC).strftime(r'%Y-%m-%d %H:%M:%S') self.run_sql( @@ -926,8 +640,6 @@ def _snapshot_and_assert_revived(self): self._revived_snapshot_datetime = self._snapshot() database = self.default_database - if self.adapter_type == 'bigquery': - database = self.adapter.quote(database) # records which weren't revived (id != 10, 11) invalidated_records = self.run_sql( diff --git a/test/integration/004_simple_snapshot_test/test_snapshot_check_cols.py b/test/integration/004_simple_snapshot_test/test_snapshot_check_cols.py index 3b49a119eb3..d92d0319ad5 100644 --- a/test/integration/004_simple_snapshot_test/test_snapshot_check_cols.py +++ b/test/integration/004_simple_snapshot_test/test_snapshot_check_cols.py @@ -1,5 +1,4 @@ from test.integration.base import DBTIntegrationTest, use_profile -import dbt.exceptions class TestSimpleSnapshotFiles(DBTIntegrationTest): @@ -35,22 +34,7 @@ def test_snapshot_check_cols_cycle(self): def assert_expected(self): self.run_dbt(['test', '--select', 'test_type:singular', '--vars', 'version: 3']) - @use_profile('snowflake') - def test__snowflake__simple_snapshot(self): - self.test_snapshot_check_cols_cycle() - self.assert_expected() - @use_profile('postgres') def test__postgres__simple_snapshot(self): self.test_snapshot_check_cols_cycle() self.assert_expected() - - @use_profile('bigquery') - def test__bigquery__simple_snapshot(self): - self.test_snapshot_check_cols_cycle() - self.assert_expected() - - @use_profile('redshift') - def test__redshift__simple_snapshot(self): - self.test_snapshot_check_cols_cycle() - self.assert_expected() diff --git a/test/integration/004_simple_snapshot_test/update_bq.sql b/test/integration/004_simple_snapshot_test/update_bq.sql deleted file mode 100644 index 5c972d8af5c..00000000000 --- a/test/integration/004_simple_snapshot_test/update_bq.sql +++ /dev/null @@ -1,78 +0,0 @@ --- insert v2 of the 11 - 21 records - -insert {database}.{schema}.snapshot_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - cast(null as timestamp) as dbt_valid_to, - updated_at as dbt_updated_at, - to_hex(md5(concat(cast(id as string), '-', first_name, '|', cast(updated_at as string)))) as dbt_scd_id -from {database}.{schema}.seed -where id >= 10 and id <= 20; - - --- insert 10 new records -insert into {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values -(21, 'Judy', 'Robinson', 'jrobinsonk@blogs.com', 'Female', '208.21.192.232', '2016-09-18 08:27:38'), -(22, 'Kevin', 'Alvarez', 'kalvarezl@buzzfeed.com', 'Male', '228.106.146.9', '2016-07-29 03:07:37'), -(23, 'Barbara', 'Carr', 'bcarrm@pen.io', 'Female', '106.165.140.17', '2015-09-24 13:27:23'), -(24, 'William', 'Watkins', 'wwatkinsn@guardian.co.uk', 'Male', '78.155.84.6', '2016-03-08 19:13:08'), -(25, 'Judy', 'Cooper', 'jcoopero@google.com.au', 'Female', '24.149.123.184', '2016-10-05 20:49:33'), -(26, 'Shirley', 'Castillo', 'scastillop@samsung.com', 'Female', '129.252.181.12', '2016-06-20 21:12:21'), -(27, 'Justin', 'Harper', 'jharperq@opera.com', 'Male', '131.172.103.218', '2016-05-21 22:56:46'), -(28, 'Marie', 'Medina', 'mmedinar@nhs.uk', 'Female', '188.119.125.67', '2015-10-08 13:44:33'), -(29, 'Kelly', 'Edwards', 'kedwardss@phoca.cz', 'Female', '47.121.157.66', '2015-09-15 06:33:37'), -(30, 'Carl', 'Coleman', 'ccolemant@wikipedia.org', 'Male', '82.227.154.83', '2016-05-26 16:46:40'); - - --- add these new records to the snapshot table -insert {database}.{schema}.snapshot_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - cast(null as timestamp) as dbt_valid_to, - updated_at as dbt_updated_at, - to_hex(md5(concat(cast(id as string), '-', first_name, '|', cast(updated_at as string)))) as dbt_scd_id -from {database}.{schema}.seed -where id > 20; - diff --git a/test/integration/005_simple_seed_test/models-bq/schema.yml b/test/integration/005_simple_seed_test/models-bq/schema.yml deleted file mode 100644 index 019a9524f0f..00000000000 --- a/test/integration/005_simple_seed_test/models-bq/schema.yml +++ /dev/null @@ -1,47 +0,0 @@ -version: 2 -seeds: -- name: seed_enabled - columns: - - name: birthday - tests: - - column_type: - type: STRING - - name: id - tests: - - column_type: - type: FLOAT64 - -- name: seed_tricky - columns: - - name: id - tests: - - column_type: - type: INT64 - - name: id_str - tests: - - column_type: - type: STRING - - name: a_bool - tests: - - column_type: - type: BOOLEAN - - name: looks_like_a_bool - tests: - - column_type: - type: STRING - - name: a_date - tests: - - column_type: - type: DATETIME - - name: looks_like_a_date - tests: - - column_type: - type: STRING - - name: relative - tests: - - column_type: - type: STRING - - name: weekday - tests: - - column_type: - type: STRING diff --git a/test/integration/005_simple_seed_test/models-rs/schema.yml b/test/integration/005_simple_seed_test/models-rs/schema.yml deleted file mode 100644 index 00a79bf9338..00000000000 --- a/test/integration/005_simple_seed_test/models-rs/schema.yml +++ /dev/null @@ -1,47 +0,0 @@ -version: 2 -seeds: -- name: seed_enabled - columns: - - name: birthday - tests: - - column_type: - type: date - - name: id - tests: - - column_type: - type: character varying(256) - -- name: seed_tricky - columns: - - name: id - tests: - - column_type: - type: integer - - name: id_str - tests: - - column_type: - type: character varying(256) - - name: a_bool - tests: - - column_type: - type: boolean - - name: looks_like_a_bool - tests: - - column_type: - type: character varying(256) - - name: a_date - tests: - - column_type: - type: timestamp without time zone - - name: looks_like_a_date - tests: - - column_type: - type: character varying(256) - - name: relative - tests: - - column_type: - type: character varying(9) - - name: weekday - tests: - - column_type: - type: character varying(8) diff --git a/test/integration/005_simple_seed_test/models-snowflake/schema.yml b/test/integration/005_simple_seed_test/models-snowflake/schema.yml deleted file mode 100644 index c8c725fc20a..00000000000 --- a/test/integration/005_simple_seed_test/models-snowflake/schema.yml +++ /dev/null @@ -1,47 +0,0 @@ -version: 2 -seeds: -- name: seed_enabled - columns: - - name: birthday - tests: - - column_type: - type: character varying(16777216) - - name: id - tests: - - column_type: - type: FLOAT - -- name: seed_tricky - columns: - - name: id - tests: - - column_type: - type: NUMBER(38,0) - - name: id_str - tests: - - column_type: - type: character varying(16777216) - - name: a_bool - tests: - - column_type: - type: BOOLEAN - - name: looks_like_a_bool - tests: - - column_type: - type: character varying(16777216) - - name: a_date - tests: - - column_type: - type: TIMESTAMP_NTZ - - name: looks_like_a_date - tests: - - column_type: - type: character varying(16777216) - - name: relative - tests: - - column_type: - type: character varying(16777216) - - name: weekday - tests: - - column_type: - type: character varying(16777216) diff --git a/test/integration/005_simple_seed_test/snowflake-data-config/seed_disabled.csv b/test/integration/005_simple_seed_test/snowflake-data-config/seed_disabled.csv deleted file mode 100644 index aec23a516de..00000000000 --- a/test/integration/005_simple_seed_test/snowflake-data-config/seed_disabled.csv +++ /dev/null @@ -1,21 +0,0 @@ -ID,FIRST_NAME,EMAIL,IP_ADDRESS,BIRTHDAY -1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31 -2,Larry,lperkins1@toplist.cz,64.210.133.162,1978-05-09 04:15:14 -3,Anna,amontgomery2@miitbeian.gov.cn,168.104.64.114,2011-10-16 04:07:57 -4,Sandra,sgeorge3@livejournal.com,229.235.252.98,1973-07-19 10:52:43 -5,Fred,fwoods4@google.cn,78.229.170.124,2012-09-30 16:38:29 -6,Stephen,shanson5@livejournal.com,182.227.157.105,1995-11-07 21:40:50 -7,William,wmartinez6@upenn.edu,135.139.249.50,1982-09-05 03:11:59 -8,Jessica,jlong7@hao123.com,203.62.178.210,1991-10-16 11:03:15 -9,Douglas,dwhite8@tamu.edu,178.187.247.1,1979-10-01 09:49:48 -10,Lisa,lcoleman9@nydailynews.com,168.234.128.249,2011-05-26 07:45:49 -11,Ralph,rfieldsa@home.pl,55.152.163.149,1972-11-18 19:06:11 -12,Louise,lnicholsb@samsung.com,141.116.153.154,2014-11-25 20:56:14 -13,Clarence,cduncanc@sfgate.com,81.171.31.133,2011-11-17 07:02:36 -14,Daniel,dfranklind@omniture.com,8.204.211.37,1980-09-13 00:09:04 -15,Katherine,klanee@auda.org.au,176.96.134.59,1997-08-22 19:36:56 -16,Billy,bwardf@wikia.com,214.108.78.85,2003-10-19 02:14:47 -17,Annie,agarzag@ocn.ne.jp,190.108.42.70,1988-10-28 15:12:35 -18,Shirley,scolemanh@fastcompany.com,109.251.164.84,1988-08-24 10:50:57 -19,Roger,rfrazieri@scribd.com,38.145.218.108,1985-12-31 15:17:15 -20,Lillian,lstanleyj@goodreads.com,47.57.236.17,1970-06-08 02:09:05 diff --git a/test/integration/005_simple_seed_test/snowflake-data-config/seed_enabled.csv b/test/integration/005_simple_seed_test/snowflake-data-config/seed_enabled.csv deleted file mode 100644 index aec23a516de..00000000000 --- a/test/integration/005_simple_seed_test/snowflake-data-config/seed_enabled.csv +++ /dev/null @@ -1,21 +0,0 @@ -ID,FIRST_NAME,EMAIL,IP_ADDRESS,BIRTHDAY -1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31 -2,Larry,lperkins1@toplist.cz,64.210.133.162,1978-05-09 04:15:14 -3,Anna,amontgomery2@miitbeian.gov.cn,168.104.64.114,2011-10-16 04:07:57 -4,Sandra,sgeorge3@livejournal.com,229.235.252.98,1973-07-19 10:52:43 -5,Fred,fwoods4@google.cn,78.229.170.124,2012-09-30 16:38:29 -6,Stephen,shanson5@livejournal.com,182.227.157.105,1995-11-07 21:40:50 -7,William,wmartinez6@upenn.edu,135.139.249.50,1982-09-05 03:11:59 -8,Jessica,jlong7@hao123.com,203.62.178.210,1991-10-16 11:03:15 -9,Douglas,dwhite8@tamu.edu,178.187.247.1,1979-10-01 09:49:48 -10,Lisa,lcoleman9@nydailynews.com,168.234.128.249,2011-05-26 07:45:49 -11,Ralph,rfieldsa@home.pl,55.152.163.149,1972-11-18 19:06:11 -12,Louise,lnicholsb@samsung.com,141.116.153.154,2014-11-25 20:56:14 -13,Clarence,cduncanc@sfgate.com,81.171.31.133,2011-11-17 07:02:36 -14,Daniel,dfranklind@omniture.com,8.204.211.37,1980-09-13 00:09:04 -15,Katherine,klanee@auda.org.au,176.96.134.59,1997-08-22 19:36:56 -16,Billy,bwardf@wikia.com,214.108.78.85,2003-10-19 02:14:47 -17,Annie,agarzag@ocn.ne.jp,190.108.42.70,1988-10-28 15:12:35 -18,Shirley,scolemanh@fastcompany.com,109.251.164.84,1988-08-24 10:50:57 -19,Roger,rfrazieri@scribd.com,38.145.218.108,1985-12-31 15:17:15 -20,Lillian,lstanleyj@goodreads.com,47.57.236.17,1970-06-08 02:09:05 diff --git a/test/integration/005_simple_seed_test/test_seed_type_override.py b/test/integration/005_simple_seed_test/test_seed_type_override.py index a6cfd372f4a..6e4b108de07 100644 --- a/test/integration/005_simple_seed_test/test_seed_type_override.py +++ b/test/integration/005_simple_seed_test/test_seed_type_override.py @@ -59,92 +59,3 @@ def test_postgres_simple_seed_with_column_override_postgres(self): results = self.run_dbt(["test"]) self.assertEqual(len(results), 10) - -class TestSimpleSeedColumnOverrideRedshift(TestSimpleSeedColumnOverride): - @property - def models(self): - return "models-rs" - - @property - def profile_config(self): - return self.redshift_profile() - - def seed_enabled_types(self): - return { - "id": "text", - "birthday": "date", - } - - def seed_tricky_types(self): - return { - 'id_str': 'text', - 'looks_like_a_bool': 'text', - 'looks_like_a_date': 'text', - } - - @use_profile('redshift') - def test_redshift_simple_seed_with_column_override_redshift(self): - results = self.run_dbt(["seed", "--show"]) - self.assertEqual(len(results), 2) - results = self.run_dbt(["test"]) - self.assertEqual(len(results), 10) - - -class TestSimpleSeedColumnOverrideSnowflake(TestSimpleSeedColumnOverride): - @property - def models(self): - return "models-snowflake" - - def seed_enabled_types(self): - return { - "id": "FLOAT", - "birthday": "TEXT", - } - - def seed_tricky_types(self): - return { - 'id_str': 'TEXT', - 'looks_like_a_bool': 'TEXT', - 'looks_like_a_date': 'TEXT', - } - - @property - def profile_config(self): - return self.snowflake_profile() - - @use_profile('snowflake') - def test_snowflake_simple_seed_with_column_override_snowflake(self): - results = self.run_dbt(["seed", "--show"]) - self.assertEqual(len(results), 2) - results = self.run_dbt(["test"]) - self.assertEqual(len(results), 10) - - -class TestSimpleSeedColumnOverrideBQ(TestSimpleSeedColumnOverride): - @property - def models(self): - return "models-bq" - - def seed_enabled_types(self): - return { - "id": "FLOAT64", - "birthday": "STRING", - } - - def seed_tricky_types(self): - return { - 'id_str': 'STRING', - 'looks_like_a_bool': 'STRING', - 'looks_like_a_date': 'STRING', - } - - @property - def profile_config(self): - return self.bigquery_profile() - - @use_profile('bigquery') - def test_bigquery_simple_seed_with_column_override_bigquery(self): - results = self.run_dbt(["seed", "--show"]) - self.assertEqual(len(results), 2) - results = self.run_dbt(["test"]) - self.assertEqual(len(results), 10) diff --git a/test/integration/005_simple_seed_test/test_simple_seed.py b/test/integration/005_simple_seed_test/test_simple_seed.py index dd825708d10..6a4aac322ff 100644 --- a/test/integration/005_simple_seed_test/test_simple_seed.py +++ b/test/integration/005_simple_seed_test/test_simple_seed.py @@ -346,8 +346,4 @@ def test_big_batched_seed(self): @use_profile('postgres') def test_postgres_big_batched_seed(self): self.test_big_batched_seed() - - @use_profile('snowflake') - def test_snowflake_big_batched_seed(self): - self.test_big_batched_seed() \ No newline at end of file diff --git a/test/integration/007_graph_selection_tests/test_graph_selection.py b/test/integration/007_graph_selection_tests/test_graph_selection.py index f0dd6188d40..8122f609da7 100644 --- a/test/integration/007_graph_selection_tests/test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_graph_selection.py @@ -109,20 +109,6 @@ def test__postgres__tags_and_children_limited(self): self.assertNotIn('users_rollup_dependency', created_models) self.assert_correct_schemas() - @use_profile('snowflake') - def test__snowflake__specific_model(self): - self.run_sql_file("seed.sql") - - results = self.run_dbt(['run', '--select', 'users']) - self.assertEqual(len(results), 1) - - self.assertTablesEqual("SEED", "USERS") - created_models = self.get_models_in_schema() - self.assertFalse('USERS_ROLLUP' in created_models) - self.assertFalse('BASE_USERS' in created_models) - self.assertFalse('EMAILS' in created_models) - self.assert_correct_schemas() - @use_profile('postgres') def test__postgres__specific_model_and_children(self): self.run_sql_file("seed.sql") @@ -139,21 +125,6 @@ def test__postgres__specific_model_and_children(self): self.assertNotIn('emails', created_models) self.assert_correct_schemas() - @use_profile('snowflake') - def test__snowflake__specific_model_and_children(self): - self.run_sql_file("seed.sql") - - results = self.run_dbt(['run', '--select', 'users+']) - self.assertEqual(len(results), 4) - - self.assertManyTablesEqual( - ["SEED", "USERS"], - ["SUMMARY_EXPECTED", "USERS_ROLLUP"] - ) - created_models = self.get_models_in_schema() - self.assertFalse('BASE_USERS' in created_models) - self.assertFalse('EMAILS' in created_models) - @use_profile('postgres') def test__postgres__specific_model_and_children_limited(self): self.run_sql_file("seed.sql") @@ -184,22 +155,6 @@ def test__postgres__specific_model_and_parents(self): self.assertFalse('emails' in created_models) self.assert_correct_schemas() - @use_profile('snowflake') - def test__snowflake__specific_model_and_parents(self): - self.run_sql_file("seed.sql") - - results = self.run_dbt(['run', '--select', '+users_rollup']) - self.assertEqual(len(results), 2) - - self.assertManyTablesEqual( - ["SEED", "USERS"], - ["SUMMARY_EXPECTED", "USERS_ROLLUP"] - ) - - created_models = self.get_models_in_schema() - self.assertFalse('BASE_USERS' in created_models) - self.assertFalse('EMAILS' in created_models) - @use_profile('postgres') def test__postgres__specific_model_and_parents_limited(self): self.run_sql_file("seed.sql") @@ -230,21 +185,6 @@ def test__postgres__specific_model_with_exclusion(self): self.assertFalse('emails' in created_models) self.assert_correct_schemas() - @use_profile('snowflake') - def test__snowflake__specific_model_with_exclusion(self): - self.run_sql_file("seed.sql") - - results = self.run_dbt( - ['run', '--select', '+users_rollup', '--exclude', 'users_rollup'] - ) - self.assertEqual(len(results), 1) - - self.assertManyTablesEqual(["SEED", "USERS"]) - created_models = self.get_models_in_schema() - self.assertFalse('BASE_USERS' in created_models) - self.assertFalse('USERS_ROLLUP' in created_models) - self.assertFalse('EMAILS' in created_models) - @use_profile('postgres') def test__postgres__locally_qualified_name(self): results = self.run_dbt(['run', '--select', 'test.subdir']) @@ -326,28 +266,6 @@ def test__postgres__more_childrens_parents(self): self.assertEqual(len(results), 2) assert sorted([r.node.name for r in results]) == ['unique_users_id', 'unique_users_rollup_gender'] - - @use_profile('snowflake') - def test__snowflake__skip_intermediate(self): - self.run_sql_file("seed.sql") - results = self.run_dbt(['run', '--select', '@models/users.sql']) - # base_users, emails, users_rollup, users_rollup_dependency - self.assertEqual(len(results), 4) - - # now re-run, skipping users_rollup - results = self.run_dbt(['run', '--select', '@users', '--exclude', 'users_rollup']) - self.assertEqual(len(results), 3) - - # make sure that users_rollup_dependency and users don't interleave - users = [r for r in results if r.node.name == 'users'][0] - dep = [r for r in results if r.node.name == 'users_rollup_dependency'][0] - user_last_end = users.timing[1].completed_at - dep_first_start = dep.timing[0].started_at - self.assertTrue( - user_last_end <= dep_first_start, - 'dependency started before its transitive parent ({} > {})'.format(user_last_end, dep_first_start) - ) - @use_profile('postgres') def test__postgres__concat(self): self.run_sql_file("seed.sql") diff --git a/test/integration/008_schema_tests_test/models-v2/bq-models/ephemeral_copy.sql b/test/integration/008_schema_tests_test/models-v2/bq-models/ephemeral_copy.sql deleted file mode 100644 index 73c62165efb..00000000000 --- a/test/integration/008_schema_tests_test/models-v2/bq-models/ephemeral_copy.sql +++ /dev/null @@ -1,8 +0,0 @@ - -{{ - config( - materialized='ephemeral' - ) -}} - -select * from {{ this.schema }}.seed diff --git a/test/integration/008_schema_tests_test/models-v2/bq-models/schema.yml b/test/integration/008_schema_tests_test/models-v2/bq-models/schema.yml deleted file mode 100644 index 15100c18e68..00000000000 --- a/test/integration/008_schema_tests_test/models-v2/bq-models/schema.yml +++ /dev/null @@ -1,30 +0,0 @@ -version: 2 - -models: - - name: ephemeral_copy - description: "An ephemeral copy of the table" - columns: - - name: id - description: "The ID" - tests: - - not_null - - unique - - name: favorite_color - tests: - - accepted_values: { values: ['blue', 'green'] } - - # this whole model should pass and run - - name: table_summary - description: "The summary table" - columns: - - name: favorite_color_copy - description: "The favorite color" - tests: - - not_null - - unique - - accepted_values: { values: ['blue', 'green'] } - - relationships: { field: favorite_color, to: ref('ephemeral_copy') } - - name: count - description: "The number of responses for this favorite color" - tests: - - not_null diff --git a/test/integration/008_schema_tests_test/models-v2/bq-models/table_summary.sql b/test/integration/008_schema_tests_test/models-v2/bq-models/table_summary.sql deleted file mode 100644 index 845ce3d58f0..00000000000 --- a/test/integration/008_schema_tests_test/models-v2/bq-models/table_summary.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - -select favorite_color as favorite_color_copy, count(*) as count -from {{ ref('ephemeral_copy') }} -group by 1 diff --git a/test/integration/008_schema_tests_test/test_schema_v2_tests.py b/test/integration/008_schema_tests_test/test_schema_v2_tests.py index 87ba64c770b..bde616cf491 100644 --- a/test/integration/008_schema_tests_test/test_schema_v2_tests.py +++ b/test/integration/008_schema_tests_test/test_schema_v2_tests.py @@ -249,55 +249,6 @@ def test_postgres_schema_tests(self): self.assertIn(result.node.name, expected_failures) -class TestBQSchemaTests(DBTIntegrationTest): - @property - def schema(self): - return "schema_tests_008" - - @property - def models(self): - return "models-v2/bq-models" - - @staticmethod - def dir(path): - return os.path.normpath( - os.path.join('models-v2', path)) - - def run_schema_validations(self): - args = FakeArgs() - - test_task = TestTask(args, self.config) - return test_task.run() - - @use_profile('bigquery') - def test_schema_tests_bigquery(self): - self.use_default_project({'data-paths': [self.dir('seed')]}) - self.assertEqual(len(self.run_dbt(['seed'])), 1) - results = self.run_dbt() - self.assertEqual(len(results), 1) - test_results = self.run_schema_validations() - self.assertEqual(len(test_results), 8) - - for result in test_results: - # assert that all deliberately failing tests actually fail - if 'failure' in result.node.name: - self.assertEqual(result.status, 'fail') - self.assertFalse(result.skipped) - self.assertTrue( - result.failures > 0, - 'test {} did not fail'.format(result.node.name) - ) - # assert that actual tests pass - else: - self.assertEqual(result.status, 'pass') - self.assertFalse(result.skipped) - self.assertEqual( - result.failures, 0, - 'test {} failed'.format(result.node.name) - ) - - self.assertEqual(sum(x.failures for x in test_results), 0) - class TestQuotedSchemaTestColumns(DBTIntegrationTest): @property def schema(self): diff --git a/test/integration/009_data_tests_test/test_data_tests.py b/test/integration/009_data_tests_test/test_data_tests.py index e09fde55d6b..aea14d6b96b 100644 --- a/test/integration/009_data_tests_test/test_data_tests.py +++ b/test/integration/009_data_tests_test/test_data_tests.py @@ -55,26 +55,3 @@ def test_postgres_data_tests(self): defined_tests = os.listdir(self.test_path) self.assertNotEqual(len(test_results), 0) self.assertEqual(len(test_results), len(defined_tests)) - - @use_profile('snowflake') - def test_snowflake_data_tests(self): - self.use_profile('snowflake') - - self.run_sql_file("seed.sql") - - results = self.run_dbt() - self.assertEqual(len(results), 1) - test_results = self.run_data_validations() - - for result in test_results: - # assert that all deliberately failing tests actually fail - if 'fail' in result.node.name: - self.assertEqual(result.status, 'fail') - self.assertFalse(result.skipped) - self.assertTrue(result.failures > 0) - - # assert that actual tests pass - else: - self.assertEqual(result.status, 'pass') - self.assertFalse(result.skipped) - self.assertEqual(result.failures, 0) diff --git a/test/integration/012_deprecation_tests/dispatch-macros/cast.sql b/test/integration/012_deprecation_tests/dispatch-macros/cast.sql index 3dc65d2d7df..924c69975bc 100644 --- a/test/integration/012_deprecation_tests/dispatch-macros/cast.sql +++ b/test/integration/012_deprecation_tests/dispatch-macros/cast.sql @@ -8,6 +8,3 @@ '{{ s }}'::text {% endmacro %} -{% macro bigquery__string_literal(s) %} - cast('{{ s }}' as string) -{% endmacro %} diff --git a/test/integration/014_hook_tests/macros/before-and-after-bq.sql b/test/integration/014_hook_tests/macros/before-and-after-bq.sql deleted file mode 100644 index 030b869b7d7..00000000000 --- a/test/integration/014_hook_tests/macros/before-and-after-bq.sql +++ /dev/null @@ -1,30 +0,0 @@ - -{% macro custom_run_hook_bq(state, target, run_started_at, invocation_id) %} - - insert into {{ target.schema }}.on_run_hook ( - state, - target_dbname, - target_host, - target_name, - target_schema, - target_type, - target_user, - target_pass, - target_threads, - run_started_at, - invocation_id - ) VALUES ( - '{{ state }}', - '{{ target.database }}', - '', {# bigquery has no host value #} - '{{ target.name }}', - '{{ target.schema }}', - '{{ target.type }}', - '', {# bigquery has no user value #} - '{{ target.get("pass", "") }}', - {{ target.threads }}, - '{{ run_started_at }}', - '{{ invocation_id }}' - ) - -{% endmacro %} diff --git a/test/integration/014_hook_tests/seed-models-bq/schema.yml b/test/integration/014_hook_tests/seed-models-bq/schema.yml deleted file mode 100644 index 21bbf202f16..00000000000 --- a/test/integration/014_hook_tests/seed-models-bq/schema.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: 2 -seeds: -- name: example_seed - columns: - - name: a - tests: - - not_null diff --git a/test/integration/014_hook_tests/seed_model_bigquery.sql b/test/integration/014_hook_tests/seed_model_bigquery.sql deleted file mode 100644 index 7093a47e898..00000000000 --- a/test/integration/014_hook_tests/seed_model_bigquery.sql +++ /dev/null @@ -1,18 +0,0 @@ - -drop table if exists {schema}.on_model_hook; - -create table {schema}.on_model_hook ( - state STRING, -- start|end - - target_dbname STRING, - target_host STRING, - target_name STRING, - target_schema STRING, - target_type STRING, - target_user STRING, - target_pass STRING, - target_threads INT64, - - run_started_at STRING, - invocation_id STRING -); diff --git a/test/integration/014_hook_tests/seed_run_bigquery.sql b/test/integration/014_hook_tests/seed_run_bigquery.sql deleted file mode 100644 index d9d5212ef5f..00000000000 --- a/test/integration/014_hook_tests/seed_run_bigquery.sql +++ /dev/null @@ -1,18 +0,0 @@ - -drop table if exists {schema}.on_run_hook; - -create table {schema}.on_run_hook ( - state STRING, -- start|end - - target_dbname STRING, - target_host STRING, - target_name STRING, - target_schema STRING, - target_type STRING, - target_user STRING, - target_pass STRING, - target_threads INT64, - - run_started_at STRING, - invocation_id STRING -); \ No newline at end of file diff --git a/test/integration/014_hook_tests/test_model_hooks_bq.py b/test/integration/014_hook_tests/test_model_hooks_bq.py deleted file mode 100644 index 581b48872af..00000000000 --- a/test/integration/014_hook_tests/test_model_hooks_bq.py +++ /dev/null @@ -1,150 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile - -MODEL_PRE_HOOK = """ - insert into {{this.schema}}.on_model_hook ( - state, - target_name, - target_schema, - target_type, - target_threads, - run_started_at, - invocation_id - ) VALUES ( - 'start', - '{{ target.name }}', - '{{ target.schema }}', - '{{ target.type }}', - {{ target.threads }}, - '{{ run_started_at }}', - '{{ invocation_id }}' - ) -""" - - -MODEL_POST_HOOK = """ - insert into {{this.schema}}.on_model_hook ( - state, - target_name, - target_schema, - target_type, - target_threads, - run_started_at, - invocation_id - ) VALUES ( - 'end', - '{{ target.name }}', - '{{ target.schema }}', - '{{ target.type }}', - {{ target.threads }}, - '{{ run_started_at }}', - '{{ invocation_id }}' - ) -""" - -class TestBigqueryPrePostModelHooks(DBTIntegrationTest): - def setUp(self): - DBTIntegrationTest.setUp(self) - self.run_sql_file("seed_model_bigquery.sql") - - self.fields = [ - 'state', - 'target_name', - 'target_schema', - 'target_threads', - 'target_type', - 'run_started_at', - 'invocation_id' - ] - - @property - def schema(self): - return "model_hooks_014" - - @property - def profile_config(self): - profile = self.bigquery_profile() - profile['test']['outputs']['default2']['threads'] = 3 - return profile - - @property - def project_config(self): - return { - 'config-version': 2, - 'macro-paths': ['macros'], - 'models': { - 'test': { - 'pre-hook': [MODEL_PRE_HOOK], - 'post-hook':[MODEL_POST_HOOK], - } - } - } - - @property - def models(self): - return "models" - - def get_ctx_vars(self, state): - field_list = ", ".join(self.fields) - query = "select {field_list} from `{schema}.on_model_hook` where state = '{state}'".format(field_list=field_list, schema=self.unique_schema(), state=state) - - vals = self.run_sql(query, fetch='all') - self.assertFalse(len(vals) == 0, 'nothing inserted into hooks table') - self.assertFalse(len(vals) > 1, 'too many rows in hooks table') - ctx = dict(zip(self.fields, vals[0])) - - return ctx - - def check_hooks(self, state): - ctx = self.get_ctx_vars(state) - - self.assertEqual(ctx['state'], state) - self.assertEqual(ctx['target_name'], 'default2') - self.assertEqual(ctx['target_schema'], self.unique_schema()) - self.assertEqual(ctx['target_threads'], 3) - self.assertEqual(ctx['target_type'], 'bigquery') - self.assertTrue(ctx['run_started_at'] is not None and len(ctx['run_started_at']) > 0, 'run_started_at was not set') - self.assertTrue(ctx['invocation_id'] is not None and len(ctx['invocation_id']) > 0, 'invocation_id was not set') - - @use_profile('bigquery') - def test_pre_and_post_model_hooks_bigquery(self): - self.run_dbt(['run']) - - self.check_hooks('start') - self.check_hooks('end') - - -class TestBigqueryPrePostModelHooksOnSeeds(DBTIntegrationTest): - @property - def schema(self): - return "model_hooks_014" - - @property - def models(self): - return "seed-models-bq" - - @property - def project_config(self): - return { - 'config-version': 2, - 'data-paths': ['data'], - 'models': {}, - 'seeds': { - 'post-hook': [ - 'insert into {{ this }} (a, b, c) VALUES (10, 11, 12)', - ], - 'quote_columns': False, - }, - } - - @use_profile('bigquery') - def test_hooks_on_seeds_bigquery(self): - res = self.run_dbt(['seed']) - self.assertEqual(len(res), 1, 'Expected exactly one item') - res = self.run_dbt(['test']) - self.assertEqual(len(res), 1, 'Expected exactly one item') - result = self.run_sql( - 'select a, b, c from `{schema}`.`example_seed` where a = 10', - fetch='all' - ) - self.assertFalse(len(result) == 0, 'nothing inserted into table by hook') - self.assertFalse(len(result) > 1, 'too many rows in table') diff --git a/test/integration/014_hook_tests/test_run_hooks_bq.py b/test/integration/014_hook_tests/test_run_hooks_bq.py deleted file mode 100644 index e0edbde1af4..00000000000 --- a/test/integration/014_hook_tests/test_run_hooks_bq.py +++ /dev/null @@ -1,102 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile - -class TestBigqueryPrePostRunHooks(DBTIntegrationTest): - - def setUp(self): - DBTIntegrationTest.setUp(self) - self.use_profile('bigquery') - self.use_default_project() - self.run_sql_file("seed_run_bigquery.sql") - - self.fields = [ - 'state', - 'target_name', - 'target_schema', - 'target_threads', - 'target_type', - 'run_started_at', - 'invocation_id' - ] - - @property - def schema(self): - return "run_hooks_014" - - @property - def profile_config(self): - profile = self.bigquery_profile() - profile['test']['outputs']['default2']['threads'] = 3 - return profile - - - @property - def project_config(self): - return { - 'config-version': 2, - 'macro-paths': ['macros'], - 'data-paths': ['data'], - - # The create and drop table statements here validate that these hooks run - # in the same order that they are defined. Drop before create is an error. - # Also check that the table does not exist below. - "on-run-start": [ - "{{ custom_run_hook_bq('start', target, run_started_at, invocation_id) }}", - "create table {{ target.schema }}.start_hook_order_test ( id INT64 )", - "drop table {{ target.schema }}.start_hook_order_test", - ], - "on-run-end": [ - "{{ custom_run_hook_bq('end', target, run_started_at, invocation_id) }}", - "create table {{ target.schema }}.end_hook_order_test ( id INT64 )", - "drop table {{ target.schema }}.end_hook_order_test", - ], - 'seeds': { - 'quote_columns': False, - }, - } - - @property - def models(self): - return "models" - - def get_ctx_vars(self, state): - field_list = ", ".join(self.fields) - query = "select {field_list} from `{schema}.on_run_hook` where state = '{state}'".format(field_list=field_list, schema=self.unique_schema(), state=state) - - vals = self.run_sql(query, fetch='all') - self.assertFalse(len(vals) == 0, 'nothing inserted into on_run_hook table') - self.assertFalse(len(vals) > 1, 'too many rows in hooks table') - ctx = dict(zip(self.fields, vals[0])) - - return ctx - - def check_hooks(self, state): - ctx = self.get_ctx_vars(state) - - self.assertEqual(ctx['state'], state) - self.assertEqual(ctx['target_name'], 'default2') - self.assertEqual(ctx['target_schema'], self.unique_schema()) - self.assertEqual(ctx['target_threads'], 3) - self.assertEqual(ctx['target_type'], 'bigquery') - - self.assertTrue(ctx['run_started_at'] is not None and len(ctx['run_started_at']) > 0, 'run_started_at was not set') - self.assertTrue(ctx['invocation_id'] is not None and len(ctx['invocation_id']) > 0, 'invocation_id was not set') - - @use_profile('bigquery') - def test_bigquery_pre_and_post_run_hooks(self): - self.run_dbt(['run']) - - self.check_hooks('start') - self.check_hooks('end') - - self.assertTableDoesNotExist("start_hook_order_test") - self.assertTableDoesNotExist("end_hook_order_test") - - @use_profile('bigquery') - def test_bigquery_pre_and_post_seed_hooks(self): - self.run_dbt(['seed']) - - self.check_hooks('start') - self.check_hooks('end') - - self.assertTableDoesNotExist("start_hook_order_test") - self.assertTableDoesNotExist("end_hook_order_test") diff --git a/test/integration/016_macro_tests/test_macros.py b/test/integration/016_macro_tests/test_macros.py index e030e2c6e9d..646e74a3b8a 100644 --- a/test/integration/016_macro_tests/test_macros.py +++ b/test/integration/016_macro_tests/test_macros.py @@ -95,20 +95,6 @@ def test_postgres_invalid_macro(self): assert "In dispatch: No macro named 'dispatch_to_nowhere' found" in str(exc.value) -class TestDispatchMacroUseParent(DBTIntegrationTest): - @property - def schema(self): - return "test_macros_016" - - @property - def models(self): - return "dispatch-inheritance-models" - - @use_profile('redshift') - def test_redshift_inherited_macro(self): - self.run_dbt(['run']) - - class TestMacroOverrideBuiltin(DBTIntegrationTest): @property def schema(self): @@ -125,7 +111,6 @@ def project_config(self): 'macro-paths': ['override-get-columns-macros'], } - @use_profile('postgres') def test_postgres_overrides(self): # the first time, the model doesn't exist diff --git a/test/integration/020_ephemeral_test/test_ephemeral.py b/test/integration/020_ephemeral_test/test_ephemeral.py index e72ebdb5040..c4df48d4f85 100644 --- a/test/integration/020_ephemeral_test/test_ephemeral.py +++ b/test/integration/020_ephemeral_test/test_ephemeral.py @@ -43,17 +43,6 @@ def test__postgres(self): expected_sql = "".join(expected_sql.split()) self.assertEqual(sql_file, expected_sql) - @use_profile('snowflake') - def test__snowflake(self): - self.run_sql_file("seed.sql") - - results = self.run_dbt() - self.assertEqual(len(results), 3) - - self.assertManyTablesEqual( - ["SEED", "DEPENDENT", "DOUBLE_DEPENDENT", "SUPER_DEPENDENT"] - ) - class TestEphemeralNested(DBTIntegrationTest): @property diff --git a/test/integration/021_concurrency_test/test_concurrency.py b/test/integration/021_concurrency_test/test_concurrency.py index 418a22590fe..1a1dab724af 100644 --- a/test/integration/021_concurrency_test/test_concurrency.py +++ b/test/integration/021_concurrency_test/test_concurrency.py @@ -37,19 +37,3 @@ def test__postgres__concurrency(self): self.assertTableDoesNotExist("skip") self.assertIn('PASS=5 WARN=0 ERROR=1 SKIP=1 TOTAL=7', output) - - @use_profile('snowflake') - def test__snowflake__concurrency(self): - self.run_sql_file("seed.sql") - - results = self.run_dbt(expect_pass=False) - self.assertEqual(len(results), 7) - - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"]) - - self.run_sql_file("update.sql") - - results = self.run_dbt(expect_pass=False) - self.assertEqual(len(results), 7) - - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"]) diff --git a/test/integration/022_bigquery_test/adapter-models/schema.yml b/test/integration/022_bigquery_test/adapter-models/schema.yml deleted file mode 100644 index 97a54b3c763..00000000000 --- a/test/integration/022_bigquery_test/adapter-models/schema.yml +++ /dev/null @@ -1,39 +0,0 @@ -version: 2 -models: -- name: test_get_columns_in_table - columns: - - name: field_1 - tests: - - not_null - - name: field_2 - tests: - - not_null - - name: field_3 - tests: - - not_null - - name: nested_field - tests: - - not_null - - name: repeated_column - tests: - - not_null -- name: test_flattened_get_columns_in_table - columns: - - name: field_1 - tests: - - not_null - - name: field_2 - tests: - - not_null - - name: field_3 - tests: - - not_null - - name: field_4 - tests: - - not_null - - name: field_5 - tests: - - not_null - - name: repeated_column - tests: - - not_null diff --git a/test/integration/022_bigquery_test/adapter-models/source.sql b/test/integration/022_bigquery_test/adapter-models/source.sql deleted file mode 100644 index 2c419f25d22..00000000000 --- a/test/integration/022_bigquery_test/adapter-models/source.sql +++ /dev/null @@ -1,41 +0,0 @@ - -with nested_base as ( - select - struct( - 'a' as field_a, - 'b' as field_b - ) as repeated_nested - - union all - - select - struct( - 'a' as field_a, - 'b' as field_b - ) as repeated_nested -), - -nested as ( - - select - array_agg(repeated_nested) as repeated_column - - from nested_base - -), - -base as ( - - select - 1 as field_1, - 2 as field_2, - 3 as field_3, - - struct( - 4 as field_4, - 5 as field_5 - ) as nested_field -) - -select * -from base, nested diff --git a/test/integration/022_bigquery_test/adapter-models/test_flattened_get_columns_in_table.sql b/test/integration/022_bigquery_test/adapter-models/test_flattened_get_columns_in_table.sql deleted file mode 100644 index 3572ad6dcb2..00000000000 --- a/test/integration/022_bigquery_test/adapter-models/test_flattened_get_columns_in_table.sql +++ /dev/null @@ -1,22 +0,0 @@ -{% set source = ref('source') %} -{% set cols = adapter.get_columns_in_relation(source) %} - -{{ log('source') }} -{{ log(source) }} - -{% set flattened = [] %} -{% for col in cols %} - {% if col.mode == 'REPEATED' %} - {% set _ = flattened.append(col) %} - {% else %} - {% set _ = flattened.extend(col.flatten()) %} - {% endif %} -{% endfor %} - -select - {% for col in flattened %} - {{ col.name }} - {% if not loop.last %}, {% endif %} - {% endfor %} - -from {{ source }} diff --git a/test/integration/022_bigquery_test/adapter-models/test_get_columns_in_table.sql b/test/integration/022_bigquery_test/adapter-models/test_get_columns_in_table.sql deleted file mode 100644 index 8135dbe9a46..00000000000 --- a/test/integration/022_bigquery_test/adapter-models/test_get_columns_in_table.sql +++ /dev/null @@ -1,10 +0,0 @@ -{% set source = ref('source') %} -{% set cols = adapter.get_columns_in_relation(source) %} - -select - {% for col in cols %} - {{ col.name }} - {% if not loop.last %}, {% endif %} - {% endfor %} - -from {{ source }} diff --git a/test/integration/022_bigquery_test/adapter-specific-models/expiring_table.sql b/test/integration/022_bigquery_test/adapter-specific-models/expiring_table.sql deleted file mode 100644 index 43258a71464..00000000000 --- a/test/integration/022_bigquery_test/adapter-specific-models/expiring_table.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as id diff --git a/test/integration/022_bigquery_test/case-sensitive-models/fUnKyCaSe.sql b/test/integration/022_bigquery_test/case-sensitive-models/fUnKyCaSe.sql deleted file mode 100644 index 1934d4cfc0c..00000000000 --- a/test/integration/022_bigquery_test/case-sensitive-models/fUnKyCaSe.sql +++ /dev/null @@ -1,5 +0,0 @@ -{{ config(materialized='incremental') }} -select 1 as id -{% if is_incremental() %} -this is a syntax error! -{% endif %} diff --git a/test/integration/022_bigquery_test/case-sensitive-schemas/model.sql b/test/integration/022_bigquery_test/case-sensitive-schemas/model.sql deleted file mode 100644 index 1934d4cfc0c..00000000000 --- a/test/integration/022_bigquery_test/case-sensitive-schemas/model.sql +++ /dev/null @@ -1,5 +0,0 @@ -{{ config(materialized='incremental') }} -select 1 as id -{% if is_incremental() %} -this is a syntax error! -{% endif %} diff --git a/test/integration/022_bigquery_test/copy-failing-models/copy_bad_materialization.sql b/test/integration/022_bigquery_test/copy-failing-models/copy_bad_materialization.sql deleted file mode 100644 index b6093645d56..00000000000 --- a/test/integration/022_bigquery_test/copy-failing-models/copy_bad_materialization.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(copy_materialization='view') }} -{{ ref('original') }} \ No newline at end of file diff --git a/test/integration/022_bigquery_test/copy-failing-models/original.sql b/test/integration/022_bigquery_test/copy-failing-models/original.sql deleted file mode 100644 index 26d9cae7b5b..00000000000 --- a/test/integration/022_bigquery_test/copy-failing-models/original.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as id \ No newline at end of file diff --git a/test/integration/022_bigquery_test/copy-models/additional.sql b/test/integration/022_bigquery_test/copy-models/additional.sql deleted file mode 100644 index 33560d6c082..00000000000 --- a/test/integration/022_bigquery_test/copy-models/additional.sql +++ /dev/null @@ -1 +0,0 @@ -select 2 as id diff --git a/test/integration/022_bigquery_test/copy-models/copy_as_incremental.sql b/test/integration/022_bigquery_test/copy-models/copy_as_incremental.sql deleted file mode 100644 index bbe8e5acd2e..00000000000 --- a/test/integration/022_bigquery_test/copy-models/copy_as_incremental.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(copy_materialization='incremental') }} -{{ ref('original') }} \ No newline at end of file diff --git a/test/integration/022_bigquery_test/copy-models/copy_as_several_tables.sql b/test/integration/022_bigquery_test/copy-models/copy_as_several_tables.sql deleted file mode 100644 index 99b04e1b381..00000000000 --- a/test/integration/022_bigquery_test/copy-models/copy_as_several_tables.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('original') }}, {{ source('test_copy_several_tables', 'additional') }} diff --git a/test/integration/022_bigquery_test/copy-models/copy_as_table.sql b/test/integration/022_bigquery_test/copy-models/copy_as_table.sql deleted file mode 100644 index 3f2fe2550af..00000000000 --- a/test/integration/022_bigquery_test/copy-models/copy_as_table.sql +++ /dev/null @@ -1 +0,0 @@ -{{ ref('original') }} diff --git a/test/integration/022_bigquery_test/copy-models/original.sql b/test/integration/022_bigquery_test/copy-models/original.sql deleted file mode 100644 index 26d9cae7b5b..00000000000 --- a/test/integration/022_bigquery_test/copy-models/original.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as id \ No newline at end of file diff --git a/test/integration/022_bigquery_test/copy-models/schema.yml b/test/integration/022_bigquery_test/copy-models/schema.yml deleted file mode 100644 index defbd47b52c..00000000000 --- a/test/integration/022_bigquery_test/copy-models/schema.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: 2 -sources: - - name: test_copy_several_tables - schema: "{{ target.schema }}" - tables: - - name: additional diff --git a/test/integration/022_bigquery_test/data/data_seed.csv b/test/integration/022_bigquery_test/data/data_seed.csv deleted file mode 100644 index afd0a31efa5..00000000000 --- a/test/integration/022_bigquery_test/data/data_seed.csv +++ /dev/null @@ -1,5 +0,0 @@ -id,dupe -1,a -2,a -3,a -4,a diff --git a/test/integration/022_bigquery_test/data/incremental_overwrite_date_expected.csv b/test/integration/022_bigquery_test/data/incremental_overwrite_date_expected.csv deleted file mode 100644 index 7454b880b7a..00000000000 --- a/test/integration/022_bigquery_test/data/incremental_overwrite_date_expected.csv +++ /dev/null @@ -1,5 +0,0 @@ -id,date_day -10,2020-01-01 -20,2020-01-01 -30,2020-01-02 -40,2020-01-02 diff --git a/test/integration/022_bigquery_test/data/incremental_overwrite_day_expected.csv b/test/integration/022_bigquery_test/data/incremental_overwrite_day_expected.csv deleted file mode 100644 index e9db40a62f3..00000000000 --- a/test/integration/022_bigquery_test/data/incremental_overwrite_day_expected.csv +++ /dev/null @@ -1,5 +0,0 @@ -id,date_time -10,2020-01-01 00:00:00 -20,2020-01-01 00:00:00 -30,2020-01-02 00:00:00 -40,2020-01-02 00:00:00 diff --git a/test/integration/022_bigquery_test/data/incremental_overwrite_range_expected.csv b/test/integration/022_bigquery_test/data/incremental_overwrite_range_expected.csv deleted file mode 100644 index 203a4a9d081..00000000000 --- a/test/integration/022_bigquery_test/data/incremental_overwrite_range_expected.csv +++ /dev/null @@ -1,5 +0,0 @@ -id,date_int -10,20200101 -20,20200101 -30,20200102 -40,20200102 diff --git a/test/integration/022_bigquery_test/data/incremental_overwrite_time_expected.csv b/test/integration/022_bigquery_test/data/incremental_overwrite_time_expected.csv deleted file mode 100644 index 50559dd51e8..00000000000 --- a/test/integration/022_bigquery_test/data/incremental_overwrite_time_expected.csv +++ /dev/null @@ -1,5 +0,0 @@ -id,date_hour -10,2020-01-01 01:00:00 -20,2020-01-01 01:00:00 -30,2020-01-01 02:00:00 -40,2020-01-01 02:00:00 diff --git a/test/integration/022_bigquery_test/data/merge_expected.csv b/test/integration/022_bigquery_test/data/merge_expected.csv deleted file mode 100644 index 0aa05500ceb..00000000000 --- a/test/integration/022_bigquery_test/data/merge_expected.csv +++ /dev/null @@ -1,7 +0,0 @@ -id,date_time -1,2020-01-01 00:00:00 -2,2020-01-01 00:00:00 -3,2020-01-01 00:00:00 -4,2020-01-02 00:00:00 -5,2020-01-02 00:00:00 -6,2020-01-02 00:00:00 \ No newline at end of file diff --git a/test/integration/022_bigquery_test/dp-models/confirmation.sql b/test/integration/022_bigquery_test/dp-models/confirmation.sql deleted file mode 100644 index 858c0e2e32f..00000000000 --- a/test/integration/022_bigquery_test/dp-models/confirmation.sql +++ /dev/null @@ -1,20 +0,0 @@ - --- This model checks to confirm that each date partition was created correctly. --- Columns day_1, day_2, and day_3 should have a value of 1, and count_days should be 3 - -with base as ( - - select - case when _PARTITIONTIME = '2018-01-01' then 1 else 0 end as day_1, - case when _PARTITIONTIME = '2018-01-02' then 1 else 0 end as day_2, - case when _PARTITIONTIME = '2018-01-03' then 1 else 0 end as day_3 - from {{ ref('partitioned') }} - -) - -select distinct - sum(day_1) over () as day_1, - sum(day_2) over () as day_2, - sum(day_3) over () as day_3, - count(*) over () as count_days -from base diff --git a/test/integration/022_bigquery_test/dp-models/confirmation_noconfig.sql b/test/integration/022_bigquery_test/dp-models/confirmation_noconfig.sql deleted file mode 100644 index 7c061f4a5de..00000000000 --- a/test/integration/022_bigquery_test/dp-models/confirmation_noconfig.sql +++ /dev/null @@ -1,20 +0,0 @@ - --- This model checks to confirm that each date partition was created correctly. --- Columns day_1, day_2, and day_3 should have a value of 1, and count_days should be 3 - -with base as ( - - select - case when _PARTITIONTIME = '2018-01-01' then 1 else 0 end as day_1, - case when _PARTITIONTIME = '2018-01-02' then 1 else 0 end as day_2, - case when _PARTITIONTIME = '2018-01-03' then 1 else 0 end as day_3 - from {{ ref('partitioned_noconfig') }} - -) - -select distinct - sum(day_1) over () as day_1, - sum(day_2) over () as day_2, - sum(day_3) over () as day_3, - count(*) over () as count_days -from base diff --git a/test/integration/022_bigquery_test/dp-models/events_20180101.sql b/test/integration/022_bigquery_test/dp-models/events_20180101.sql deleted file mode 100644 index 9a8f54d5bcb..00000000000 --- a/test/integration/022_bigquery_test/dp-models/events_20180101.sql +++ /dev/null @@ -1,4 +0,0 @@ - -{{ config(materialized='table') }} - -select 1 as id diff --git a/test/integration/022_bigquery_test/dp-models/events_20180102.sql b/test/integration/022_bigquery_test/dp-models/events_20180102.sql deleted file mode 100644 index 63bfcdc13fe..00000000000 --- a/test/integration/022_bigquery_test/dp-models/events_20180102.sql +++ /dev/null @@ -1,4 +0,0 @@ - -{{ config(materialized='table') }} - -select 2 as id diff --git a/test/integration/022_bigquery_test/dp-models/events_20180103.sql b/test/integration/022_bigquery_test/dp-models/events_20180103.sql deleted file mode 100644 index 09a9f02c7b1..00000000000 --- a/test/integration/022_bigquery_test/dp-models/events_20180103.sql +++ /dev/null @@ -1,4 +0,0 @@ - -{{ config(materialized='table') }} - -select 3 as id diff --git a/test/integration/022_bigquery_test/dp-models/partitioned.sql b/test/integration/022_bigquery_test/dp-models/partitioned.sql deleted file mode 100644 index 5d77021d30c..00000000000 --- a/test/integration/022_bigquery_test/dp-models/partitioned.sql +++ /dev/null @@ -1,16 +0,0 @@ - -{{ - config( - materialized='table', - partitions=['20180101', '20180102', '20180103'], - verbose=True - ) -}} - --- Hack to make sure our events models run first. --- In practice, these would be source data --- {{ ref('events_20180101') }} --- {{ ref('events_20180102') }} --- {{ ref('events_20180103') }} - -select * from `{{ this.schema }}`.`{{ date_sharded_table('events_') }}` diff --git a/test/integration/022_bigquery_test/dp-models/partitioned_noconfig.sql b/test/integration/022_bigquery_test/dp-models/partitioned_noconfig.sql deleted file mode 100644 index c199d37a94c..00000000000 --- a/test/integration/022_bigquery_test/dp-models/partitioned_noconfig.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Hack to make sure our events models run first. --- In practice, these would be source data --- {{ ref('events_20180101') }} --- {{ ref('events_20180102') }} --- {{ ref('events_20180103') }} - -select * from `{{ this.schema }}`.`{{ date_sharded_table('events_') }}` diff --git a/test/integration/022_bigquery_test/dp-models/partitioned_simple.sql b/test/integration/022_bigquery_test/dp-models/partitioned_simple.sql deleted file mode 100644 index af65072bead..00000000000 --- a/test/integration/022_bigquery_test/dp-models/partitioned_simple.sql +++ /dev/null @@ -1,14 +0,0 @@ - -{{ - config( - materialized='table', - partition_date='20180101', - verbose=True - ) -}} - --- Hack to make sure our events models run first. --- In practice, these would be source data --- {{ ref('events_20180101') }} - -select * from `{{ this.schema }}`.`events_20180101` diff --git a/test/integration/022_bigquery_test/dp-models/schema.yml b/test/integration/022_bigquery_test/dp-models/schema.yml deleted file mode 100644 index 3daf468dc0a..00000000000 --- a/test/integration/022_bigquery_test/dp-models/schema.yml +++ /dev/null @@ -1,33 +0,0 @@ -version: 2 -models: -- name: partitioned_simple - columns: - - name: id - tests: - - not_null - - unique -- name: confirmation - columns: &conf_columns - - name: cast(count_days as string) - tests: - - accepted_values: - values: - - 3 - - name: cast(day_1 as string) - tests: - - accepted_values: - values: - - 1 - - name: cast(day_2 as string) - tests: - - accepted_values: - values: - - 1 - - name: cast(day_3 as string) - tests: - - accepted_values: - values: - - 1 - -- name: confirmation_noconfig - columns: *conf_columns diff --git a/test/integration/022_bigquery_test/execution-project-models/model.sql b/test/integration/022_bigquery_test/execution-project-models/model.sql deleted file mode 100644 index 43258a71464..00000000000 --- a/test/integration/022_bigquery_test/execution-project-models/model.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as id diff --git a/test/integration/022_bigquery_test/execution-project-models/schema.yml b/test/integration/022_bigquery_test/execution-project-models/schema.yml deleted file mode 100644 index 390165a8610..00000000000 --- a/test/integration/022_bigquery_test/execution-project-models/schema.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: 2 -models: -- name: model - description: | - I'm testing the profile execution_project - tests: - - project_for_job_id: - region: region-us - project_id: "{{ project_id}}" - unique_schema_id: "{{ unique_schema_id }}" \ No newline at end of file diff --git a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_merge_range.sql b/test/integration/022_bigquery_test/incremental-strategy-models/incremental_merge_range.sql deleted file mode 100644 index 581007c012c..00000000000 --- a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_merge_range.sql +++ /dev/null @@ -1,46 +0,0 @@ - -{{ - config( - materialized="incremental", - unique_key="id", - cluster_by="id", - partition_by={ - "field": "id", - "data_type": "int64", - "range": { - "start": 1, - "end": 10, - "interval": 1 - } - } - ) -}} - - -with data as ( - - {% if not is_incremental() %} - - select 1 as id, cast('2020-01-01' as datetime) as date_time union all - select 2 as id, cast('2020-01-01' as datetime) as date_time union all - select 3 as id, cast('2020-01-01' as datetime) as date_time union all - select 4 as id, cast('2020-01-01' as datetime) as date_time - - {% else %} - - select 1 as id, cast('2020-01-01' as datetime) as date_time union all - select 2 as id, cast('2020-01-01' as datetime) as date_time union all - select 3 as id, cast('2020-01-01' as datetime) as date_time union all - select 4 as id, cast('2020-01-02' as datetime) as date_time union all - select 5 as id, cast('2020-01-02' as datetime) as date_time union all - select 6 as id, cast('2020-01-02' as datetime) as date_time - - {% endif %} - -) - -select * from data - -{% if is_incremental() %} -where id >= (select max(id) from {{ this }}) -{% endif %} diff --git a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_merge_time.sql b/test/integration/022_bigquery_test/incremental-strategy-models/incremental_merge_time.sql deleted file mode 100644 index 7629a990465..00000000000 --- a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_merge_time.sql +++ /dev/null @@ -1,42 +0,0 @@ - -{{ - config( - materialized="incremental", - unique_key="id", - cluster_by="id", - partition_by={ - "field": "date_time", - "data_type": "datetime" - } - ) -}} - - - -with data as ( - - {% if not is_incremental() %} - - select 1 as id, cast('2020-01-01' as datetime) as date_time union all - select 2 as id, cast('2020-01-01' as datetime) as date_time union all - select 3 as id, cast('2020-01-01' as datetime) as date_time union all - select 4 as id, cast('2020-01-01' as datetime) as date_time - - {% else %} - - select 1 as id, cast('2020-01-01' as datetime) as date_time union all - select 2 as id, cast('2020-01-01' as datetime) as date_time union all - select 3 as id, cast('2020-01-01' as datetime) as date_time union all - select 4 as id, cast('2020-01-02' as datetime) as date_time union all - select 5 as id, cast('2020-01-02' as datetime) as date_time union all - select 6 as id, cast('2020-01-02' as datetime) as date_time - - {% endif %} - -) - -select * from data - -{% if is_incremental() %} -where date_time > (select max(date_time) from {{ this }}) -{% endif %} diff --git a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_date.sql b/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_date.sql deleted file mode 100644 index df67599efe1..00000000000 --- a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_date.sql +++ /dev/null @@ -1,41 +0,0 @@ - -{{ - config( - materialized="incremental", - incremental_strategy='insert_overwrite', - cluster_by="id", - partition_by={ - "field": "date_day", - "data_type": "date" - } - ) -}} - - -with data as ( - - {% if not is_incremental() %} - - select 1 as id, cast('2020-01-01' as date) as date_day union all - select 2 as id, cast('2020-01-01' as date) as date_day union all - select 3 as id, cast('2020-01-01' as date) as date_day union all - select 4 as id, cast('2020-01-01' as date) as date_day - - {% else %} - - -- we want to overwrite the 4 records in the 2020-01-01 partition - -- with the 2 records below, but add two more in the 2020-01-02 partition - select 10 as id, cast('2020-01-01' as date) as date_day union all - select 20 as id, cast('2020-01-01' as date) as date_day union all - select 30 as id, cast('2020-01-02' as date) as date_day union all - select 40 as id, cast('2020-01-02' as date) as date_day - - {% endif %} - -) - -select * from data - -{% if is_incremental() %} -where date_day >= _dbt_max_partition -{% endif %} diff --git a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_day.sql b/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_day.sql deleted file mode 100644 index 3434a5b47d3..00000000000 --- a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_day.sql +++ /dev/null @@ -1,41 +0,0 @@ - -{{ - config( - materialized="incremental", - incremental_strategy='insert_overwrite', - cluster_by="id", - partition_by={ - "field": "date_time", - "data_type": "datetime" - } - ) -}} - - -with data as ( - - {% if not is_incremental() %} - - select 1 as id, cast('2020-01-01' as datetime) as date_time union all - select 2 as id, cast('2020-01-01' as datetime) as date_time union all - select 3 as id, cast('2020-01-01' as datetime) as date_time union all - select 4 as id, cast('2020-01-01' as datetime) as date_time - - {% else %} - - -- we want to overwrite the 4 records in the 2020-01-01 partition - -- with the 2 records below, but add two more in the 2020-01-02 partition - select 10 as id, cast('2020-01-01' as datetime) as date_time union all - select 20 as id, cast('2020-01-01' as datetime) as date_time union all - select 30 as id, cast('2020-01-02' as datetime) as date_time union all - select 40 as id, cast('2020-01-02' as datetime) as date_time - - {% endif %} - -) - -select * from data - -{% if is_incremental() %} -where date_time >= _dbt_max_partition -{% endif %} diff --git a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_partitions.sql b/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_partitions.sql deleted file mode 100644 index ef6cf88836b..00000000000 --- a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_partitions.sql +++ /dev/null @@ -1,42 +0,0 @@ - -{{ - config( - materialized="incremental", - incremental_strategy='insert_overwrite', - cluster_by="id", - partitions=["'2020-01-01'","'2020-01-02'"], - partition_by={ - "field": "date_day", - "data_type": "date" - } - ) -}} - - -with data as ( - - {% if not is_incremental() %} - - select 1 as id, cast('2020-01-01' as date) as date_day union all - select 2 as id, cast('2020-01-01' as date) as date_day union all - select 3 as id, cast('2020-01-01' as date) as date_day union all - select 4 as id, cast('2020-01-01' as date) as date_day - - {% else %} - - -- we want to overwrite the 4 records in the 2020-01-01 partition - -- with the 2 records below, but add two more in the 2020-01-02 partition - select 10 as id, cast('2020-01-01' as date) as date_day union all - select 20 as id, cast('2020-01-01' as date) as date_day union all - select 30 as id, cast('2020-01-02' as date) as date_day union all - select 40 as id, cast('2020-01-02' as date) as date_day - - {% endif %} - -) - -select * from data - -{% if is_incremental() %} -where date_day in ({{ config.get("partitions") | join(",") }}) -{% endif %} diff --git a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_range.sql b/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_range.sql deleted file mode 100644 index 44d3ffc863c..00000000000 --- a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_range.sql +++ /dev/null @@ -1,46 +0,0 @@ - -{{ - config( - materialized="incremental", - incremental_strategy='insert_overwrite', - cluster_by="id", - partition_by={ - "field": "date_int", - "data_type": "int64", - "range": { - "start": 20200101, - "end": 20200110, - "interval": 1 - } - } - ) -}} - - -with data as ( - - {% if not is_incremental() %} - - select 1 as id, 20200101 as date_int union all - select 2 as id, 20200101 as date_int union all - select 3 as id, 20200101 as date_int union all - select 4 as id, 20200101 as date_int - - {% else %} - - -- we want to overwrite the 4 records in the 20200101 partition - -- with the 2 records below, but add two more in the 20200102 partition - select 10 as id, 20200101 as date_int union all - select 20 as id, 20200101 as date_int union all - select 30 as id, 20200102 as date_int union all - select 40 as id, 20200102 as date_int - - {% endif %} - -) - -select * from data - -{% if is_incremental() %} -where date_int >= _dbt_max_partition -{% endif %} diff --git a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_time.sql b/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_time.sql deleted file mode 100644 index 77f90189478..00000000000 --- a/test/integration/022_bigquery_test/incremental-strategy-models/incremental_overwrite_time.sql +++ /dev/null @@ -1,42 +0,0 @@ - -{{ - config( - materialized="incremental", - incremental_strategy='insert_overwrite', - cluster_by="id", - partition_by={ - "field": "date_hour", - "data_type": "datetime", - "granularity": "hour" - } - ) -}} - - -with data as ( - - {% if not is_incremental() %} - - select 1 as id, cast('2020-01-01 01:00:00' as datetime) as date_hour union all - select 2 as id, cast('2020-01-01 01:00:00' as datetime) as date_hour union all - select 3 as id, cast('2020-01-01 01:00:00' as datetime) as date_hour union all - select 4 as id, cast('2020-01-01 01:00:00' as datetime) as date_hour - - {% else %} - - -- we want to overwrite the 4 records in the 2020-01-01 01:00:00 partition - -- with the 2 records below, but add two more in the 2020-01-00 02:00:00 partition - select 10 as id, cast('2020-01-01 01:00:00' as datetime) as date_hour union all - select 20 as id, cast('2020-01-01 01:00:00' as datetime) as date_hour union all - select 30 as id, cast('2020-01-01 02:00:00' as datetime) as date_hour union all - select 40 as id, cast('2020-01-01 02:00:00' as datetime) as date_hour - - {% endif %} - -) - -select * from data - -{% if is_incremental() %} -where date_hour >= _dbt_max_partition -{% endif %} diff --git a/test/integration/022_bigquery_test/location-models/model.sql b/test/integration/022_bigquery_test/location-models/model.sql deleted file mode 100644 index 43258a71464..00000000000 --- a/test/integration/022_bigquery_test/location-models/model.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as id diff --git a/test/integration/022_bigquery_test/macros/partition_metadata.sql b/test/integration/022_bigquery_test/macros/partition_metadata.sql deleted file mode 100644 index 4a4131c9e34..00000000000 --- a/test/integration/022_bigquery_test/macros/partition_metadata.sql +++ /dev/null @@ -1,19 +0,0 @@ - -{% test number_partitions(model, expected) %} - - {%- set result = get_partitions_metadata(model) %} - - {% if result %} - {% set partitions = result.columns['partition_id'].values() %} - {% else %} - {% set partitions = () %} - {% endif %} - - {% set actual = partitions | length %} - {% set success = 1 if model and actual == expected else 0 %} - - select 'Expected {{ expected }}, but got {{ actual }}' as validation_error - from (select true) - where {{ success }} = 0 - -{% endtest %} diff --git a/test/integration/022_bigquery_test/macros/test_creation.sql b/test/integration/022_bigquery_test/macros/test_creation.sql deleted file mode 100644 index f0f2389196f..00000000000 --- a/test/integration/022_bigquery_test/macros/test_creation.sql +++ /dev/null @@ -1,18 +0,0 @@ - -{% test was_materialized(model, name, type) %} - - {#-- don't run this query in the parsing step #} - {%- if model -%} - {%- set table = adapter.get_relation(database=model.database, schema=model.schema, - identifier=model.name) -%} - {%- else -%} - {%- set table = {} -%} - {%- endif -%} - - {% if table %} - select '{{ table.type }} does not match expected value {{ type }}' - from (select true) - where '{{ table.type }}' != '{{ type }}' - {% endif %} - -{% endtest %} diff --git a/test/integration/022_bigquery_test/macros/test_int_inference.sql b/test/integration/022_bigquery_test/macros/test_int_inference.sql deleted file mode 100644 index a1ab1c8a900..00000000000 --- a/test/integration/022_bigquery_test/macros/test_int_inference.sql +++ /dev/null @@ -1,36 +0,0 @@ - -{% macro assert_eq(value, expected, msg) %} - {% if value != expected %} - {% do exceptions.raise_compiler_error(msg ~ value) %} - {% endif %} -{% endmacro %} - - -{% macro test_int_inference() %} - - {% set sql %} - select - 0 as int_0, - 1 as int_1, - 2 as int_2 - {% endset %} - - {% set result = run_query(sql) %} - {% do assert_eq((result | length), 1, 'expected 1 result, got ') %} - {% set actual_0 = result[0]['int_0'] %} - {% set actual_1 = result[0]['int_1'] %} - {% set actual_2 = result[0]['int_2'] %} - - {% do assert_eq(actual_0, 0, 'expected expected actual_0 to be 0, it was ') %} - {% do assert_eq((actual_0 | string), '0', 'expected string form of actual_0 to be 0, it was ') %} - {% do assert_eq((actual_0 * 2), 0, 'expected actual_0 * 2 to be 0, it was ') %} {# not 00 #} - - {% do assert_eq(actual_1, 1, 'expected actual_1 to be 1, it was ') %} - {% do assert_eq((actual_1 | string), '1', 'expected string form of actual_1 to be 1, it was ') %} - {% do assert_eq((actual_1 * 2), 2, 'expected actual_1 * 2 to be 2, it was ') %} {# not 11 #} - - {% do assert_eq(actual_2, 2, 'expected actual_2 to be 2, it was ') %} - {% do assert_eq((actual_2 | string), '2', 'expected string form of actual_2 to be 2, it was ') %} - {% do assert_eq((actual_2 * 2), 4, 'expected actual_2 * 2 to be 4, it was ') %} {# not 22 #} - -{% endmacro %} diff --git a/test/integration/022_bigquery_test/macros/test_project_for_job_id.sql b/test/integration/022_bigquery_test/macros/test_project_for_job_id.sql deleted file mode 100644 index 327f4e2e2bc..00000000000 --- a/test/integration/022_bigquery_test/macros/test_project_for_job_id.sql +++ /dev/null @@ -1,7 +0,0 @@ -{% test project_for_job_id(model, region, unique_schema_id, project_id) %} -select 1 -from `region-us`.INFORMATION_SCHEMA.JOBS_BY_PROJECT -where date(creation_time) = current_date - and job_project = {{project_id}} - and destination_table.dataset_id = {{unique_schema_id}} -{% endtest %} diff --git a/test/integration/022_bigquery_test/macros/wrapped_macros.sql b/test/integration/022_bigquery_test/macros/wrapped_macros.sql deleted file mode 100644 index fc66f5a9588..00000000000 --- a/test/integration/022_bigquery_test/macros/wrapped_macros.sql +++ /dev/null @@ -1,43 +0,0 @@ -{% macro my_create_schema(db_name, schema_name) %} - {% if not execute %} - {% do return(None) %} - {% endif %} - {% set relation = api.Relation.create(database=db_name, schema=schema_name).without_identifier() %} - {% do create_schema(relation) %} -{% endmacro %} - -{% macro my_drop_schema(db_name, schema_name) %} - {% if not execute %} - {% do return(None) %} - {% endif %} - {% set relation = api.Relation.create(database=db_name, schema=schema_name).without_identifier() %} - {% do drop_schema(relation) %} -{% endmacro %} - - -{% macro my_create_table_as(db_name, schema_name, table_name) %} - {% if not execute %} - {% do return(None) %} - {% endif %} - {% set relation = api.Relation.create(database=db_name, schema=schema_name, identifier=table_name) %} - {% do run_query(create_table_as(false, relation, 'select 1 as id')) %} -{% endmacro %} - - -{% macro ensure_one_relation_in(db_name, schema_name) %} - {% if not execute %} - {% do return(None) %} - {% endif %} - {% set relation = api.Relation.create(database=db_name, schema=schema_name).without_identifier() %} - {% set results = list_relations_without_caching(relation) %} - {% set rlen = (results | length) %} - {% if rlen != 1 %} - {% do exceptions.raise_compiler_error('Incorect number of results (expected 1): ' ~ rlen) %} - {% endif %} - {% set result = results[0] %} - {% set columns = get_columns_in_relation(result) %} - {% set clen = (columns | length) %} - {% if clen != 1 %} - {% do exceptions.raise_compiler_error('Incorrect number of columns (expected 1): ' ~ clen) %} - {% endif %} -{% endmacro %} diff --git a/test/integration/022_bigquery_test/models/clustered_model.sql b/test/integration/022_bigquery_test/models/clustered_model.sql deleted file mode 100644 index 1e0987cccf1..00000000000 --- a/test/integration/022_bigquery_test/models/clustered_model.sql +++ /dev/null @@ -1,10 +0,0 @@ - -{{ - config( - materialized = "table", - partition_by = {"field": "updated_at", "data_type": "date"}, - cluster_by = "dupe", - ) -}} - -select * from {{ ref('view_model') }} diff --git a/test/integration/022_bigquery_test/models/fUnKyCaSe.sql b/test/integration/022_bigquery_test/models/fUnKyCaSe.sql deleted file mode 100644 index 43258a71464..00000000000 --- a/test/integration/022_bigquery_test/models/fUnKyCaSe.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as id diff --git a/test/integration/022_bigquery_test/models/labeled_model.sql b/test/integration/022_bigquery_test/models/labeled_model.sql deleted file mode 100644 index b8847e81963..00000000000 --- a/test/integration/022_bigquery_test/models/labeled_model.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ - config( - materialized = "table", - labels = {'town': 'fish', 'analytics': 'yes'} - ) -}} - -select * from {{ ref('view_model') }} diff --git a/test/integration/022_bigquery_test/models/multi_clustered_model.sql b/test/integration/022_bigquery_test/models/multi_clustered_model.sql deleted file mode 100644 index c2093d6d6b9..00000000000 --- a/test/integration/022_bigquery_test/models/multi_clustered_model.sql +++ /dev/null @@ -1,10 +0,0 @@ - -{{ - config( - materialized = "table", - partition_by = {"field": "updated_at", "data_type": "date"}, - cluster_by = ["dupe","id"], - ) -}} - -select * from {{ ref('view_model') }} diff --git a/test/integration/022_bigquery_test/models/partitioned_model.sql b/test/integration/022_bigquery_test/models/partitioned_model.sql deleted file mode 100644 index 0f30a2185a9..00000000000 --- a/test/integration/022_bigquery_test/models/partitioned_model.sql +++ /dev/null @@ -1,9 +0,0 @@ - -{{ - config( - materialized = "table", - partition_by = {'field': 'updated_at', 'data_type': 'date'}, - ) -}} - -select * from {{ ref('view_model') }} diff --git a/test/integration/022_bigquery_test/models/schema.yml b/test/integration/022_bigquery_test/models/schema.yml deleted file mode 100644 index 68625ffc68d..00000000000 --- a/test/integration/022_bigquery_test/models/schema.yml +++ /dev/null @@ -1,60 +0,0 @@ -version: 2 -models: -- name: view_model - description: | - View model description "with double quotes" - and with 'single quotes' as welll as other; - '''abc123''' - reserved -- characters - -- - /* comment */ - columns: - - name: dupe - tests: - - unique - - name: id - tests: - - not_null - - unique - - name: updated_at - tests: - - not_null - tests: - - was_materialized: - name: view_model - type: view -- name: table_model - description: | - View model description "with double quotes" - and with 'single quotes' as welll as other; - '''abc123''' - reserved -- characters - -- - /* comment */ - columns: - - name: id - tests: - - not_null - tests: - - was_materialized: - name: table_model - type: table -- name: fUnKyCaSe - columns: - - name: id - tests: - - not_null - - unique - tests: - - was_materialized: - name: fUnKyCaSe - type: view - - -sources: - - name: raw - project: "{{ target.database }}" - dataset: "{{ target.schema }}" - tables: - - name: seed - identifier: data_seed diff --git a/test/integration/022_bigquery_test/models/sql_header_model.sql b/test/integration/022_bigquery_test/models/sql_header_model.sql deleted file mode 100644 index e49d82c4bc0..00000000000 --- a/test/integration/022_bigquery_test/models/sql_header_model.sql +++ /dev/null @@ -1,14 +0,0 @@ -{{ config(materialized="table") }} - -{# This will fail if it is not extracted correctly #} -{% call set_sql_header(config) %} - CREATE TEMPORARY FUNCTION a_to_b(str STRING) - RETURNS STRING AS ( - CASE - WHEN LOWER(str) = 'a' THEN 'b' - ELSE str - END - ); -{% endcall %} - -select a_to_b(dupe) as dupe from {{ ref('view_model') }} diff --git a/test/integration/022_bigquery_test/models/sql_header_model_incr.sql b/test/integration/022_bigquery_test/models/sql_header_model_incr.sql deleted file mode 100644 index 2b9597686bb..00000000000 --- a/test/integration/022_bigquery_test/models/sql_header_model_incr.sql +++ /dev/null @@ -1,17 +0,0 @@ - -{{ config(materialized="incremental") }} - -{# This will fail if it is not extracted correctly #} -{% call set_sql_header(config) %} - DECLARE int_var INT64 DEFAULT 42; - - CREATE TEMPORARY FUNCTION a_to_b(str STRING) - RETURNS STRING AS ( - CASE - WHEN LOWER(str) = 'a' THEN 'b' - ELSE str - END - ); -{% endcall %} - -select a_to_b(dupe) as dupe from {{ ref('view_model') }} diff --git a/test/integration/022_bigquery_test/models/sql_header_model_incr_insert_overwrite.sql b/test/integration/022_bigquery_test/models/sql_header_model_incr_insert_overwrite.sql deleted file mode 100644 index 0cafb848932..00000000000 --- a/test/integration/022_bigquery_test/models/sql_header_model_incr_insert_overwrite.sql +++ /dev/null @@ -1,33 +0,0 @@ - -{# - Ensure that the insert overwrite incremental strategy - works correctly when a UDF is used in a sql_header. The - failure mode here is that dbt might inject the UDF header - twice: once for the `create table` and then again for the - merge statement. -#} - -{{ config( - materialized="incremental", - incremental_strategy='insert_overwrite', - partition_by={"field": "dt", "data_type": "date"} -) }} - -{# This will fail if it is not extracted correctly #} -{% call set_sql_header(config) %} - DECLARE int_var INT64 DEFAULT 42; - - CREATE TEMPORARY FUNCTION a_to_b(str STRING) - RETURNS STRING AS ( - CASE - WHEN LOWER(str) = 'a' THEN 'b' - ELSE str - END - ); -{% endcall %} - -select - current_date() as dt, - a_to_b(dupe) as dupe - -from {{ ref('view_model') }} diff --git a/test/integration/022_bigquery_test/models/sql_header_model_incr_insert_overwrite_static.sql b/test/integration/022_bigquery_test/models/sql_header_model_incr_insert_overwrite_static.sql deleted file mode 100644 index 4d760a0fd96..00000000000 --- a/test/integration/022_bigquery_test/models/sql_header_model_incr_insert_overwrite_static.sql +++ /dev/null @@ -1,32 +0,0 @@ - -{# - Ensure that the insert overwrite incremental strategy - works correctly when a UDF is used in a sql_header. The - failure mode here is that dbt might inject the UDF header - twice: once for the `create table` and then again for the - merge statement. -#} - -{{ config( - materialized="incremental", - incremental_strategy='insert_overwrite', - partition_by={"field": "dt", "data_type": "date"}, - partitions=["'2020-01-1'"] -) }} - -{# This will fail if it is not extracted correctly #} -{% call set_sql_header(config) %} - CREATE TEMPORARY FUNCTION a_to_b(str STRING) - RETURNS STRING AS ( - CASE - WHEN LOWER(str) = 'a' THEN 'b' - ELSE str - END - ); -{% endcall %} - -select - cast('2020-01-01' as date) as dt, - a_to_b(dupe) as dupe - -from {{ ref('view_model') }} diff --git a/test/integration/022_bigquery_test/models/table_model.sql b/test/integration/022_bigquery_test/models/table_model.sql deleted file mode 100644 index c748eb687ef..00000000000 --- a/test/integration/022_bigquery_test/models/table_model.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ - config( - materialized = "table", - persist_docs={ "relation": true, "columns": true, "schema": true } - ) -}} - -select * from {{ ref('view_model') }} diff --git a/test/integration/022_bigquery_test/models/view_model.sql b/test/integration/022_bigquery_test/models/view_model.sql deleted file mode 100644 index bacdaf874ea..00000000000 --- a/test/integration/022_bigquery_test/models/view_model.sql +++ /dev/null @@ -1,14 +0,0 @@ -{{ - config( - materialized = "view", - persist_docs={ "relation": true, "columns": true, "schema": true } - ) -}} - - -select - id, - current_date as updated_at, - dupe - -from {{ source('raw', 'seed') }} diff --git a/test/integration/022_bigquery_test/partition-models/my_model.sql b/test/integration/022_bigquery_test/partition-models/my_model.sql deleted file mode 100644 index 554db093282..00000000000 --- a/test/integration/022_bigquery_test/partition-models/my_model.sql +++ /dev/null @@ -1,15 +0,0 @@ - - -{{ - config( - materialized="table", - partition_by=var('partition_by'), - cluster_by=var('cluster_by'), - partition_expiration_days=var('partition_expiration_days'), - require_partition_filter=var('require_partition_filter') - ) -}} - -select 1 as id, 'dr. bigquery' as name, current_timestamp() as cur_time, current_date() as cur_date -union all -select 2 as id, 'prof. bigquery' as name, current_timestamp() as cur_time, current_date() as cur_date diff --git a/test/integration/022_bigquery_test/partition-models/schema.yml b/test/integration/022_bigquery_test/partition-models/schema.yml deleted file mode 100644 index 208701eaaff..00000000000 --- a/test/integration/022_bigquery_test/partition-models/schema.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: 2 -models: -- name: my_model - tests: - - number_partitions: - expected: "{{ var('expected', 1) }}" diff --git a/test/integration/022_bigquery_test/test_bigquery_adapter_functions.py b/test/integration/022_bigquery_test/test_bigquery_adapter_functions.py deleted file mode 100644 index 480dca07d93..00000000000 --- a/test/integration/022_bigquery_test/test_bigquery_adapter_functions.py +++ /dev/null @@ -1,77 +0,0 @@ -from test.integration.base import DBTIntegrationTest, FakeArgs, use_profile -import yaml - - -class TestBigqueryAdapterFunctions(DBTIntegrationTest): - - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "adapter-models" - - @property - def profile_config(self): - return self.bigquery_profile() - - @use_profile('bigquery') - def test__bigquery_adapter_functions(self): - results = self.run_dbt() - self.assertEqual(len(results), 3) - - test_results = self.run_dbt(['test']) - - self.assertTrue(len(test_results) > 0) - for result in test_results: - self.assertEqual(result.status, 'pass') - self.assertFalse(result.skipped) - self.assertEqual(result.failures, 0) - - -class TestBigqueryAdapterMacros(DBTIntegrationTest): - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "models" - - def _create_schema_named(self, database, schema): - # do not create the initial schema. We'll do this ourselves! - pass - - @use_profile('bigquery') - def test__bigquery_run_create_drop_schema(self): - schema_args = yaml.safe_dump({ - 'db_name': self.default_database, - 'schema_name': self.unique_schema(), - }) - self.run_dbt( - ['run-operation', 'my_create_schema', '--args', schema_args]) - relation_args = yaml.safe_dump({ - 'db_name': self.default_database, - 'schema_name': self.unique_schema(), - 'table_name': 'some_table', - }) - self.run_dbt(['run-operation', 'my_create_table_as', - '--args', relation_args]) - # exercise list_relations_without_caching and get_columns_in_relation - self.run_dbt( - ['run-operation', 'ensure_one_relation_in', '--args', schema_args]) - # now to drop the schema - schema_relation = self.adapter.Relation.create( - database=self.default_database, schema=self.unique_schema()).without_identifier() - with self.adapter.connection_named('test'): - results = self.adapter.list_relations_without_caching( - schema_relation) - assert len(results) == 1 - - self.run_dbt( - ['run-operation', 'my_drop_schema', '--args', schema_args]) - with self.adapter.connection_named('test'): - results = self.adapter.list_relations_without_caching( - schema_relation) - assert len(results) == 0 diff --git a/test/integration/022_bigquery_test/test_bigquery_adapter_specific.py b/test/integration/022_bigquery_test/test_bigquery_adapter_specific.py deleted file mode 100644 index 244d7ce56a0..00000000000 --- a/test/integration/022_bigquery_test/test_bigquery_adapter_specific.py +++ /dev/null @@ -1,40 +0,0 @@ -""""Test adapter specific config options.""" -from pprint import pprint - -from test.integration.base import DBTIntegrationTest, use_profile -import textwrap -import yaml - - -class TestBigqueryAdapterSpecific(DBTIntegrationTest): - - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "adapter-specific-models" - - @property - def profile_config(self): - return self.bigquery_profile() - - @property - def project_config(self): - return yaml.safe_load(textwrap.dedent('''\ - config-version: 2 - models: - test: - materialized: table - expiring_table: - hours_to_expiration: 4 - ''')) - - @use_profile('bigquery') - def test_bigquery_hours_to_expiration(self): - _, stdout = self.run_dbt_and_capture(['--debug', 'run']) - - self.assertIn( - 'expiration_timestamp=TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL ' - '4 hour)', stdout) diff --git a/test/integration/022_bigquery_test/test_bigquery_case_sensitive.py b/test/integration/022_bigquery_test/test_bigquery_case_sensitive.py deleted file mode 100644 index cecd85c7238..00000000000 --- a/test/integration/022_bigquery_test/test_bigquery_case_sensitive.py +++ /dev/null @@ -1,34 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile - - -class TestCaseSensitiveModelBigQueryRun(DBTIntegrationTest): - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "case-sensitive-models" - - @use_profile('bigquery') - def test__bigquery_double_run_fails(self): - results = self.run_dbt() - self.assertEqual(len(results), 1) - self.run_dbt(expect_pass=False) - - -class TestCaseSensitiveSchemaBigQueryRun(TestCaseSensitiveModelBigQueryRun): - # same test, but the schema is funky instead of the model name - @property - def schema(self): - return "BigQuerY_test_022" - - def unique_schema(self): - schema = self.schema - - to_return = "{}_{}".format(self.prefix, schema) - return to_return - - @property - def models(self): - return "case-sensitive-schemas" diff --git a/test/integration/022_bigquery_test/test_bigquery_changing_partitions.py b/test/integration/022_bigquery_test/test_bigquery_changing_partitions.py deleted file mode 100644 index dff169adc98..00000000000 --- a/test/integration/022_bigquery_test/test_bigquery_changing_partitions.py +++ /dev/null @@ -1,183 +0,0 @@ -from test.integration.base import DBTIntegrationTest, FakeArgs, use_profile -import json - - -class TestChangingPartitions(DBTIntegrationTest): - - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "partition-models" - - def run_changes(self, before, after): - results = self.run_dbt(['run', '--vars', json.dumps(before)]) - self.assertEqual(len(results), 1) - - results = self.run_dbt(['run', '--vars', json.dumps(after)]) - self.assertEqual(len(results), 1) - - def test_partitions(self, expected): - test_results = self.run_dbt(['test', '--vars', json.dumps(expected)]) - - for result in test_results: - self.assertEqual(result.status, 'pass') - self.assertFalse(result.skipped) - self.assertEqual(result.failures, 0) - - @use_profile('bigquery') - def test_bigquery_add_partition(self): - before = {"partition_by": None, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - after = {"partition_by": {'field': 'cur_time', 'data_type': 'timestamp'}, - "cluster_by": None, - 'partition_expiration_days': 7, - 'require_partition_filter': True} - self.run_changes(before, after) - self.test_partitions({"expected": 1}) - - @use_profile('bigquery') - def test_bigquery_add_partition_year(self): - before = {"partition_by": None, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - after = {"partition_by": {'field': 'cur_time', 'data_type': 'timestamp', 'granularity': 'year'}, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - self.run_changes(before, after) - self.test_partitions({"expected": 1}) - - @use_profile('bigquery') - def test_bigquery_add_partition_month(self): - before = {"partition_by": None, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - after = {"partition_by": {'field': 'cur_time', 'data_type': 'timestamp', 'granularity': 'month'}, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - self.run_changes(before, after) - self.test_partitions({"expected": 1}) - - @use_profile('bigquery') - def test_bigquery_add_partition_hour(self): - before = {"partition_by": None, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - after = {"partition_by": {'field': 'cur_time', 'data_type': 'timestamp', 'granularity': 'hour'}, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - self.run_changes(before, after) - self.test_partitions({"expected": 1}) - - @use_profile('bigquery') - def test_bigquery_add_partition_hour(self): - before = {"partition_by": {'field': 'cur_time', 'data_type': 'timestamp', 'granularity': 'day'}, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - after = {"partition_by": {'field': 'cur_time', 'data_type': 'timestamp', 'granularity': 'hour'}, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - self.run_changes(before, after) - self.test_partitions({"expected": 1}) - - @use_profile('bigquery') - def test_bigquery_remove_partition(self): - before = {"partition_by": {'field': 'cur_time', 'data_type': 'timestamp'}, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - after = {"partition_by": None, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - self.run_changes(before, after) - - @use_profile('bigquery') - def test_bigquery_change_partitions(self): - before = {"partition_by": {'field': 'cur_time', 'data_type': 'timestamp'}, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - after = {"partition_by": {'field': "cur_date"}, - "cluster_by": None, - 'partition_expiration_days': 7, - 'require_partition_filter': True} - self.run_changes(before, after) - self.test_partitions({"expected": 1}) - self.run_changes(after, before) - self.test_partitions({"expected": 1}) - - @use_profile('bigquery') - def test_bigquery_change_partitions_from_int(self): - before = {"partition_by": {"field": "id", "data_type": "int64", "range": {"start": 0, "end": 10, "interval": 1}}, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - after = {"partition_by": {"field": "cur_date", "data_type": "date"}, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - self.run_changes(before, after) - self.test_partitions({"expected": 1}) - self.run_changes(after, before) - self.test_partitions({"expected": 2}) - - @use_profile('bigquery') - def test_bigquery_add_clustering(self): - before = {"partition_by": {'field': 'cur_time', 'data_type': 'timestamp'}, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - after = {"partition_by": {'field': "cur_date"}, - "cluster_by": "id", - 'partition_expiration_days': None, - 'require_partition_filter': None} - self.run_changes(before, after) - - @use_profile('bigquery') - def test_bigquery_remove_clustering(self): - before = {"partition_by": {'field': 'cur_time', 'data_type': 'timestamp'}, - "cluster_by": "id", - 'partition_expiration_days': None, - 'require_partition_filter': None} - after = {"partition_by": {'field': "cur_date"}, - "cluster_by": None, - 'partition_expiration_days': None, - 'require_partition_filter': None} - self.run_changes(before, after) - - @use_profile('bigquery') - def test_bigquery_change_clustering(self): - before = {"partition_by": {'field': 'cur_time', 'data_type': 'timestamp'}, - "cluster_by": "id", - 'partition_expiration_days': None, - 'require_partition_filter': None} - after = {"partition_by": {'field': "cur_date"}, - "cluster_by": "name", - 'partition_expiration_days': None, - 'require_partition_filter': None} - self.run_changes(before, after) - - @use_profile('bigquery') - def test_bigquery_change_clustering_strict(self): - before = {'partition_by': {'field': 'cur_time', 'data_type': 'timestamp'}, - 'cluster_by': 'id', - 'partition_expiration_days': None, - 'require_partition_filter': None} - after = {'partition_by': {'field': 'cur_date', 'data_type': 'date'}, - 'cluster_by': 'name', - 'partition_expiration_days': None, - 'require_partition_filter': None} - self.run_changes(before, after) diff --git a/test/integration/022_bigquery_test/test_bigquery_copy_failing_models.py b/test/integration/022_bigquery_test/test_bigquery_copy_failing_models.py deleted file mode 100644 index da5316da8cc..00000000000 --- a/test/integration/022_bigquery_test/test_bigquery_copy_failing_models.py +++ /dev/null @@ -1,36 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile -import textwrap -import yaml - - -class TestBigqueryCopyTableFails(DBTIntegrationTest): - - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "copy-failing-models" - - @property - def profile_config(self): - return self.bigquery_profile() - - @property - def project_config(self): - return yaml.safe_load(textwrap.dedent('''\ - config-version: 2 - models: - test: - original: - materialized: table - copy_bad_materialization: - materialized: copy - ''')) - - @use_profile('bigquery') - def test__bigquery_copy_table_fails(self): - results = self.run_dbt(expect_pass=False) - self.assertEqual(len(results), 2) - self.assertEqual(results[1].status, 'error') diff --git a/test/integration/022_bigquery_test/test_bigquery_copy_models.py b/test/integration/022_bigquery_test/test_bigquery_copy_models.py deleted file mode 100644 index 4f223dc68a8..00000000000 --- a/test/integration/022_bigquery_test/test_bigquery_copy_models.py +++ /dev/null @@ -1,41 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile -import textwrap -import yaml - - -class TestBigqueryCopyTable(DBTIntegrationTest): - - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "copy-models" - - @property - def profile_config(self): - return self.bigquery_profile() - - @property - def project_config(self): - return yaml.safe_load(textwrap.dedent('''\ - config-version: 2 - models: - test: - original: - materialized: table - additional: - materialized: table - copy_as_table: - materialized: copy - copy_as_several_tables: - materialized: copy - copy_as_incremental: - materialized: copy - ''')) - - @use_profile('bigquery') - def test__bigquery_copy_table(self): - results = self.run_dbt() - self.assertEqual(len(results), 5) diff --git a/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py b/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py deleted file mode 100644 index c787d8d3f6f..00000000000 --- a/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py +++ /dev/null @@ -1,46 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile -import textwrap -import yaml - - -class TestBigqueryDatePartitioning(DBTIntegrationTest): - - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "dp-models" - - @property - def profile_config(self): - return self.bigquery_profile() - - @property - def project_config(self): - return yaml.safe_load(textwrap.dedent('''\ - config-version: 2 - models: - test: - partitioned_noconfig: - materialized: table - partitions: - - 20180101 - - 20180102 - - 20180103 - verbose: true - ''')) - - @use_profile('bigquery') - def test__bigquery_date_partitioning(self): - results = self.run_dbt() - self.assertEqual(len(results), 8) - - test_results = self.run_dbt(['test']) - - self.assertTrue(len(test_results) > 0) - for result in test_results: - self.assertEqual(result.status, 'pass') - self.assertFalse(result.skipped) - self.assertEqual(result.failures, 0) diff --git a/test/integration/022_bigquery_test/test_bigquery_execution_project.py b/test/integration/022_bigquery_test/test_bigquery_execution_project.py deleted file mode 100644 index 3924a3bf1e9..00000000000 --- a/test/integration/022_bigquery_test/test_bigquery_execution_project.py +++ /dev/null @@ -1,23 +0,0 @@ -import os -from test.integration.base import DBTIntegrationTest, use_profile - - -class TestAlternateExecutionProjectBigQueryRun(DBTIntegrationTest): - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "execution-project-models" - - @use_profile('bigquery') - def test__bigquery_execute_project(self): - results = self.run_dbt(['run', '--models', 'model']) - self.assertEqual(len(results), 1) - execution_project = os.environ['BIGQUERY_TEST_ALT_DATABASE'] - self.run_dbt(['test', - '--target', 'alternate', - '--vars', '{ project_id: %s, unique_schema_id: %s }' - % (execution_project, self.unique_schema())], - expect_pass=False) diff --git a/test/integration/022_bigquery_test/test_bigquery_location_change.py b/test/integration/022_bigquery_test/test_bigquery_location_change.py deleted file mode 100644 index d88f29b2855..00000000000 --- a/test/integration/022_bigquery_test/test_bigquery_location_change.py +++ /dev/null @@ -1,31 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile -import os - - -class TestBigqueryErrorHandling(DBTIntegrationTest): - def setUp(self): - self.valid_location = os.getenv('DBT_TEST_BIGQUERY_INITIAL_LOCATION', 'US') - self.invalid_location = os.getenv('DBT_TEST_BIGQUERY_BAD_LOCATION', 'northamerica-northeast1') - self.location = self.valid_location - super().setUp() - - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "location-models" - - def bigquery_profile(self): - result = super().bigquery_profile() - result['test']['outputs']['default2']['location'] = self.location - return result - - @use_profile('bigquery') - def test_bigquery_location_invalid(self): - self.run_dbt() - self.location = self.invalid_location - self.use_profile('bigquery') - _, stdout = self.run_dbt_and_capture(expect_pass=False) - assert 'Query Job SQL Follows' not in stdout diff --git a/test/integration/022_bigquery_test/test_bigquery_query_results.py b/test/integration/022_bigquery_test/test_bigquery_query_results.py deleted file mode 100644 index c45ca275faf..00000000000 --- a/test/integration/022_bigquery_test/test_bigquery_query_results.py +++ /dev/null @@ -1,24 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile - - -class TestBaseBigQueryResults(DBTIntegrationTest): - - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'macro-paths': ['macros'], - } - - @use_profile('bigquery') - def test__bigquery_type_inference(self): - result = self.run_dbt(['run-operation', 'test_int_inference']) - self.assertTrue(result.success) diff --git a/test/integration/022_bigquery_test/test_bigquery_repeated_records.py b/test/integration/022_bigquery_test/test_bigquery_repeated_records.py deleted file mode 100644 index a631cc2deab..00000000000 --- a/test/integration/022_bigquery_test/test_bigquery_repeated_records.py +++ /dev/null @@ -1,68 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile -import json - -class TestBaseBigQueryRun(DBTIntegrationTest): - - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'macro-paths': ['macros'], - } - - @use_profile('bigquery') - def test__bigquery_fetch_nested_records(self): - sql = """ - select - struct( - cast('Michael' as string) as fname, - cast('Stonebreaker' as string) as lname - ) as user, - [ - struct(1 as val_1, cast(2.12 as numeric) as val_2), - struct(3 as val_1, cast(4.83 as numeric) as val_2) - ] as val - - union all - - select - struct( - cast('Johnny' as string) as fname, - cast('Brickmaker' as string) as lname - ) as user, - [ - struct(7 as val_1, cast(8 as numeric) as val_2), - struct(9 as val_1, cast(null as numeric) as val_2) - ] as val - """ - - - status, res = self.adapter.execute(sql, fetch=True) - - self.assertEqual(len(res), 2, "incorrect row count") - - expected = { - "user": [ - '{"fname": "Michael", "lname": "Stonebreaker"}', - '{"fname": "Johnny", "lname": "Brickmaker"}' - ], - "val": [ - '[{"val_1": 1, "val_2": 2.12}, {"val_1": 3, "val_2": 4.83}]', - '[{"val_1": 7, "val_2": 8}, {"val_1": 9, "val_2": null}]' - ] - } - - for i, key in enumerate(expected): - line = "row {} for key {} ({} vs {})".format(i, key, expected[key][i], res[i][key]) - # py2 serializes these in an unordered way - deserialize to compare - v1 = expected[key][i] - v2 = res[i][key] - self.assertEqual(json.loads(v1), json.loads(v2), line) diff --git a/test/integration/022_bigquery_test/test_bigquery_update_columns.py b/test/integration/022_bigquery_test/test_bigquery_update_columns.py deleted file mode 100644 index 93179a19dc9..00000000000 --- a/test/integration/022_bigquery_test/test_bigquery_update_columns.py +++ /dev/null @@ -1,85 +0,0 @@ -import os - -from test.integration.base import DBTIntegrationTest, use_profile - - -class TestBigqueryUpdateColumnPolicyTag(DBTIntegrationTest): - - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "update-column-policy-tag" - - @property - def project_config(self): - return { - 'config-version': 2, - 'vars': { - 'policy_tag': self.policy_tag - } - } - - @property - def policy_tag(self): - return os.environ.get('BIGQUERY_POLICY_TAG') - - @use_profile('bigquery') - def test__bigquery_update_column_policy_tag(self): - if self.policy_tag: - results = self.run_dbt(['run', '--models', 'policy_tag_table']) - self.assertEqual(len(results), 1) - - with self.get_connection() as conn: - client = conn.handle - - table = client.get_table( - self.adapter.connections.get_bq_table( - self.default_database, self.unique_schema(), 'policy_tag_table') - ) - - for schema_field in table.schema: - self.assertEquals(schema_field.policy_tags.names, - (self.policy_tag,)) - - -class TestBigqueryUpdateColumnDescription(DBTIntegrationTest): - - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "update-column-description" - - @property - def project_config(self): - return { - 'config-version': 2, - 'vars': { - 'field_description': self.field_description - } - } - - @property - def field_description(self): - return 'this is a field' - - @use_profile('bigquery') - def test__bigquery_update_column_description(self): - results = self.run_dbt(['run', '--models', 'description_table']) - self.assertEqual(len(results), 1) - - with self.get_connection() as conn: - client = conn.handle - - table = client.get_table( - self.adapter.connections.get_bq_table( - self.default_database, self.unique_schema(), 'description_table') - ) - - for schema_field in table.schema: - self.assertEquals(schema_field.description, self.field_description) diff --git a/test/integration/022_bigquery_test/test_incremental_strategies.py b/test/integration/022_bigquery_test/test_incremental_strategies.py deleted file mode 100644 index ed9c3f29d7b..00000000000 --- a/test/integration/022_bigquery_test/test_incremental_strategies.py +++ /dev/null @@ -1,29 +0,0 @@ -from test.integration.base import DBTIntegrationTest, FakeArgs, use_profile - -class TestBigQueryScripting(DBTIntegrationTest): - - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "incremental-strategy-models" - - @use_profile('bigquery') - def test__bigquery_assert_incrementals(self): - results = self.run_dbt() - self.assertEqual(len(results), 7) - - results = self.run_dbt() - self.assertEqual(len(results), 7) - - results = self.run_dbt(['seed']) - - self.assertTablesEqual('incremental_merge_range', 'merge_expected') - self.assertTablesEqual('incremental_merge_time', 'merge_expected') - self.assertTablesEqual('incremental_overwrite_time', 'incremental_overwrite_time_expected') - self.assertTablesEqual('incremental_overwrite_date', 'incremental_overwrite_date_expected') - self.assertTablesEqual('incremental_overwrite_partitions', 'incremental_overwrite_date_expected') - self.assertTablesEqual('incremental_overwrite_day', 'incremental_overwrite_day_expected') - self.assertTablesEqual('incremental_overwrite_range', 'incremental_overwrite_range_expected') diff --git a/test/integration/022_bigquery_test/test_simple_bigquery_view.py b/test/integration/022_bigquery_test/test_simple_bigquery_view.py deleted file mode 100644 index 3e8c47829cd..00000000000 --- a/test/integration/022_bigquery_test/test_simple_bigquery_view.py +++ /dev/null @@ -1,71 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile -import random -import time - - -class TestBaseBigQueryRun(DBTIntegrationTest): - - @property - def schema(self): - return "bigquery_test_022" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'data-paths': ['data'], - 'macro-paths': ['macros'], - 'seeds': { - 'quote_columns': False, - }, - } - - @property - def profile_config(self): - return self.bigquery_profile() - - def assert_nondupes_pass(self): - # The 'dupe' model should fail, but all others should pass - test_results = self.run_dbt(['test'], expect_pass=False) - - for result in test_results: - if 'dupe' in result.node.name: - self.assertEqual(result.status, 'fail') - self.assertFalse(result.skipped) - self.assertTrue(result.failures > 0) - - # assert that actual tests pass - else: - self.assertEqual(result.status, 'pass') - self.assertFalse(result.skipped) - self.assertEqual(result.failures, 0) - - -class TestSimpleBigQueryRun(TestBaseBigQueryRun): - - @use_profile('bigquery') - def test__bigquery_simple_run(self): - # make sure seed works twice. Full-refresh is a no-op - self.run_dbt(['seed']) - self.run_dbt(['seed', '--full-refresh']) - results = self.run_dbt() - # Bump expected number of results when adding new model - self.assertEqual(len(results), 11) - self.assert_nondupes_pass() - - -class TestUnderscoreBigQueryRun(TestBaseBigQueryRun): - prefix = "_test{}{:04}".format(int(time.time()), random.randint(0, 9999)) - - @use_profile('bigquery') - def test_bigquery_run_twice(self): - self.run_dbt(['seed']) - results = self.run_dbt() - self.assertEqual(len(results), 11) - results = self.run_dbt() - self.assertEqual(len(results), 11) - self.assert_nondupes_pass() diff --git a/test/integration/022_bigquery_test/update-column-description/description_table.sql b/test/integration/022_bigquery_test/update-column-description/description_table.sql deleted file mode 100644 index 7110ac8b0f5..00000000000 --- a/test/integration/022_bigquery_test/update-column-description/description_table.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ - config( - materialized='table', - persist_docs={ 'columns': true } - ) -}} - -select - 1 field diff --git a/test/integration/022_bigquery_test/update-column-description/description_table.yml b/test/integration/022_bigquery_test/update-column-description/description_table.yml deleted file mode 100644 index 28d68f0076b..00000000000 --- a/test/integration/022_bigquery_test/update-column-description/description_table.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: 2 - -models: -- name: description_table - columns: - - name: field - description: '{{ var("field_description") }}' diff --git a/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.sql b/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.sql deleted file mode 100644 index 7110ac8b0f5..00000000000 --- a/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ - config( - materialized='table', - persist_docs={ 'columns': true } - ) -}} - -select - 1 field diff --git a/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.yml b/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.yml deleted file mode 100644 index 01a76c50dde..00000000000 --- a/test/integration/022_bigquery_test/update-column-policy-tag/policy_tag_table.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: 2 - -models: -- name: policy_tag_table - columns: - - name: field - policy_tags: - - '{{ var("policy_tag") }}' diff --git a/test/integration/024_custom_schema_test/custom-db-macros/custom_db.sql b/test/integration/024_custom_schema_test/custom-db-macros/custom_db.sql deleted file mode 100644 index bb9717490de..00000000000 --- a/test/integration/024_custom_schema_test/custom-db-macros/custom_db.sql +++ /dev/null @@ -1,10 +0,0 @@ - -{% macro generate_database_name(database_name, node) %} - {% if database_name == 'alt' %} - {{ env_var('SNOWFLAKE_TEST_ALT_DATABASE') }} - {% elif database_name %} - {{ database_name }} - {% else %} - {{ target.database }} - {% endif %} -{% endmacro %} diff --git a/test/integration/024_custom_schema_test/db-models/view_1.sql b/test/integration/024_custom_schema_test/db-models/view_1.sql deleted file mode 100644 index 501c773e8f0..00000000000 --- a/test/integration/024_custom_schema_test/db-models/view_1.sql +++ /dev/null @@ -1,3 +0,0 @@ - - -select * from {{ target.schema }}.seed diff --git a/test/integration/024_custom_schema_test/db-models/view_2.sql b/test/integration/024_custom_schema_test/db-models/view_2.sql deleted file mode 100644 index 7bec9b2052f..00000000000 --- a/test/integration/024_custom_schema_test/db-models/view_2.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(database='alt') }} -select * from {{ ref('view_1') }} diff --git a/test/integration/024_custom_schema_test/db-models/view_3.sql b/test/integration/024_custom_schema_test/db-models/view_3.sql deleted file mode 100644 index 825a672c59b..00000000000 --- a/test/integration/024_custom_schema_test/db-models/view_3.sql +++ /dev/null @@ -1,30 +0,0 @@ - -{{ config(database='alt', materialized='table') }} - - -with v1 as ( - - select * from {{ ref('view_1') }} - -), - -v2 as ( - - select * from {{ ref('view_2') }} - -), - -combined as ( - - select last_name from v1 - union all - select last_name from v2 - -) - -select - last_name, - count(*) as count - -from combined -group by 1 diff --git a/test/integration/024_custom_schema_test/test_custom_database.py b/test/integration/024_custom_schema_test/test_custom_database.py deleted file mode 100644 index 477d0a1e3a5..00000000000 --- a/test/integration/024_custom_schema_test/test_custom_database.py +++ /dev/null @@ -1,39 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile - - -class TestOverrideDatabase(DBTIntegrationTest): - setup_alternate_db = True - - @property - def schema(self): - return "custom_schema_024" - - @property - def models(self): - return "db-models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'macro-paths': ['custom-db-macros'], - } - - @use_profile('snowflake') - def test_snowflake_override_generate_db_name(self): - self.run_sql_file('seed.sql') - self.assertTableDoesExist('SEED', schema=self.unique_schema(), database=self.default_database) - self.assertTableDoesExist('AGG', schema=self.unique_schema(), database=self.default_database) - - results = self.run_dbt() - self.assertEqual(len(results), 3) - - self.assertTableDoesExist('VIEW_1', schema=self.unique_schema(), database=self.default_database) - self.assertTableDoesExist('VIEW_2', schema=self.unique_schema(), database=self.alternative_database) - self.assertTableDoesExist('VIEW_3', schema=self.unique_schema(), database=self.alternative_database) - - # not overridden - self.assertTablesEqual('SEED', 'VIEW_1', table_b_db=self.default_database) - # overridden - self.assertTablesEqual('SEED', 'VIEW_2', table_b_db=self.alternative_database) - self.assertTablesEqual('AGG', 'VIEW_3', table_b_db=self.alternative_database) diff --git a/test/integration/024_custom_schema_test/test_custom_schema.py b/test/integration/024_custom_schema_test/test_custom_schema.py index 50bbe78ccd3..fca0f9c5c4b 100644 --- a/test/integration/024_custom_schema_test/test_custom_schema.py +++ b/test/integration/024_custom_schema_test/test_custom_schema.py @@ -131,59 +131,6 @@ def test__postgres__custom_schema_with_prefix(self): self.assertTablesEqual("agg", "view_3", schema, self.xf_schema()) -class TestCustomProjectSchemaWithPrefixSnowflake(DBTIntegrationTest): - def setUp(self): - super().setUp() - self._created_schemas.add( - self._get_schema_fqn(self.default_database, self.v1_schema()) - ) - self._created_schemas.add( - self._get_schema_fqn(self.default_database, self.v2_schema()) - ) - self._created_schemas.add( - self._get_schema_fqn(self.default_database, self.xf_schema()) - ) - - @property - def schema(self): - return "sf_custom_prefix_024" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - "models": { - "schema": "dbt_test" - } - } - - def v1_schema(self): - return f"{self.unique_schema()}_DBT_TEST" - - def v2_schema(self): - return f"{self.unique_schema()}_CUSTOM" - - def xf_schema(self): - return f"{self.unique_schema()}_TEST" - - @use_profile('snowflake') - def test__snowflake__custom_schema_with_prefix(self): - self.run_sql_file("seed.sql") - - results = self.run_dbt() - self.assertEqual(len(results), 3) - - schema = self.unique_schema().upper() - - self.assertTablesEqual("SEED", "VIEW_1", schema, self.v1_schema()) - self.assertTablesEqual("SEED", "VIEW_2", schema, self.v2_schema()) - self.assertTablesEqual("AGG", "VIEW_3", schema, self.xf_schema()) - - class TestCustomSchemaWithCustomMacro(DBTIntegrationTest): def setUp(self): super().setUp() diff --git a/test/integration/026_aliases_test/macros/cast.sql b/test/integration/026_aliases_test/macros/cast.sql index 584492077ac..69fd56b0976 100644 --- a/test/integration/026_aliases_test/macros/cast.sql +++ b/test/integration/026_aliases_test/macros/cast.sql @@ -7,7 +7,3 @@ {% macro default__string_literal(s) %} '{{ s }}'::text {% endmacro %} - -{% macro bigquery__string_literal(s) %} - cast('{{ s }}' as string) -{% endmacro %} diff --git a/test/integration/026_aliases_test/test_aliases.py b/test/integration/026_aliases_test/test_aliases.py index fd6a3521299..b618a7a8478 100644 --- a/test/integration/026_aliases_test/test_aliases.py +++ b/test/integration/026_aliases_test/test_aliases.py @@ -33,18 +33,6 @@ def test__alias_model_name_postgres(self): self.assertEqual(len(results), 4) self.run_dbt(['test']) - @use_profile('bigquery') - def test__alias_model_name_bigquery(self): - results = self.run_dbt(['run']) - self.assertEqual(len(results), 4) - self.run_dbt(['test']) - - @use_profile('snowflake') - def test__alias_model_name_snowflake(self): - results = self.run_dbt(['run']) - self.assertEqual(len(results), 4) - self.run_dbt(['test']) - class TestAliasErrors(DBTIntegrationTest): @property @@ -129,12 +117,3 @@ def project_config(self): }, } } - - @use_profile('bigquery') - def test__bigquery_same_alias_succeeds_in_different_schemas(self): - results = self.run_dbt(['run']) - self.assertEqual(len(results), 2) - res = self.run_dbt(['test']) - - # Make extra sure the tests ran - self.assertTrue(len(res) > 0) diff --git a/test/integration/029_docs_generate_tests/bq_models/clustered.sql b/test/integration/029_docs_generate_tests/bq_models/clustered.sql deleted file mode 100644 index 744d2ecb298..00000000000 --- a/test/integration/029_docs_generate_tests/bq_models/clustered.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ - config( - materialized='table', - partition_by={'field': 'updated_at', 'data_type': 'date'}, - cluster_by=['first_name'] - ) -}} - -select id,first_name,email,ip_address,DATE(updated_at) as updated_at from {{ ref('seed') }} diff --git a/test/integration/029_docs_generate_tests/bq_models/multi_clustered.sql b/test/integration/029_docs_generate_tests/bq_models/multi_clustered.sql deleted file mode 100644 index e47df02f9b0..00000000000 --- a/test/integration/029_docs_generate_tests/bq_models/multi_clustered.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ - config( - materialized='table', - partition_by={'field': 'updated_at', 'data_type': 'date'}, - cluster_by=['first_name','email'] - ) -}} - -select id,first_name,email,ip_address,DATE(updated_at) as updated_at from {{ ref('seed') }} diff --git a/test/integration/029_docs_generate_tests/bq_models/nested_table.sql b/test/integration/029_docs_generate_tests/bq_models/nested_table.sql deleted file mode 100644 index 22f9048d629..00000000000 --- a/test/integration/029_docs_generate_tests/bq_models/nested_table.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - -select - 1 as field_1, - 2 as field_2, - 3 as field_3, - - struct( - 5 as field_5, - 6 as field_6 - ) as nested_field diff --git a/test/integration/029_docs_generate_tests/bq_models/nested_view.sql b/test/integration/029_docs_generate_tests/bq_models/nested_view.sql deleted file mode 100644 index 29e7b4a2d92..00000000000 --- a/test/integration/029_docs_generate_tests/bq_models/nested_view.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ - config( - materialized='view' - ) -}} - -select * from {{ ref('nested_table') }} diff --git a/test/integration/029_docs_generate_tests/bq_models/schema.yml b/test/integration/029_docs_generate_tests/bq_models/schema.yml deleted file mode 100644 index 87ebd8c7179..00000000000 --- a/test/integration/029_docs_generate_tests/bq_models/schema.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: 2 - -models: - - name: nested_view - description: "The test model" - columns: - - name: field_1 - description: The first field - - name: field_2 - description: The second field - - name: field_3 - description: The third field - - name: nested_field.field_4 - description: The first nested field - - name: nested_field.field_5 - description: The second nested field - - name: clustered - description: "A clustered and partitioned copy of the test model" - columns: - - name: id - description: The user id - - name: first_name - description: The user's name - - name: email - description: The user's email - - name: ip_address - description: The user's IP address - - name: updated_at - description: When the user was updated - - name: multi_clustered - description: "A clustered and partitioned copy of the test model, clustered on multiple columns" - columns: - - name: id - description: The user id - - name: first_name - description: The user's name - - name: email - description: The user's email - - name: ip_address - description: The user's IP address - - name: updated_at - description: When the user was updated - - diff --git a/test/integration/029_docs_generate_tests/bq_models_noschema/disabled.sql b/test/integration/029_docs_generate_tests/bq_models_noschema/disabled.sql deleted file mode 100644 index e4368a859ca..00000000000 --- a/test/integration/029_docs_generate_tests/bq_models_noschema/disabled.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(disabled=true, schema='notrealnotreal') }} -select 1 as id diff --git a/test/integration/029_docs_generate_tests/bq_models_noschema/model.sql b/test/integration/029_docs_generate_tests/bq_models_noschema/model.sql deleted file mode 100644 index 2fb872e8446..00000000000 --- a/test/integration/029_docs_generate_tests/bq_models_noschema/model.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(schema=var('extra_schema')) }} -select 1 as id diff --git a/test/integration/029_docs_generate_tests/models/second_model.sql b/test/integration/029_docs_generate_tests/models/second_model.sql index be1462db47e..37f83155ce9 100644 --- a/test/integration/029_docs_generate_tests/models/second_model.sql +++ b/test/integration/029_docs_generate_tests/models/second_model.sql @@ -1,12 +1,7 @@ -{%- if adapter.type() == 'snowflake' -%} - {%- set schema_suffix = 'TEST' -%} -{%- else -%} - {%- set schema_suffix = 'test' -%} -{%- endif -%} {{ config( materialized='view', - schema=schema_suffix, + schema='test', ) }} diff --git a/test/integration/029_docs_generate_tests/rs_models/model.sql b/test/integration/029_docs_generate_tests/rs_models/model.sql deleted file mode 100644 index 8ca73c7a0f4..00000000000 --- a/test/integration/029_docs_generate_tests/rs_models/model.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ - config( - materialized='view', bind=False - ) -}} - -select * from {{ ref('seed') }} diff --git a/test/integration/029_docs_generate_tests/rs_models/schema.yml b/test/integration/029_docs_generate_tests/rs_models/schema.yml deleted file mode 100644 index 37dd397c85a..00000000000 --- a/test/integration/029_docs_generate_tests/rs_models/schema.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: 2 - -models: - - name: model - description: "The test model" - columns: - - name: id - description: The user ID number - - name: first_name - description: The user's first name - - name: email - description: The user's email - - name: ip_address - description: The user's IP address - - name: updated_at - description: The last time this user's email was updated diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 1db684c906c..e35d9840def 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -99,17 +99,12 @@ class TestDocsGenerate(DBTIntegrationTest): setup_alternate_db = True def adapter_case(self, value): - if self.adapter_type == 'snowflake': - return value.upper() - else: - return value.lower() + return value.lower() def setUp(self): super().setUp() self.maxDiff = None self.alternate_schema = self.unique_schema() + '_test' - if self.adapter_type == 'snowflake': - self.alternate_schema = self.alternate_schema.upper() self._created_schemas.add(self.alternate_schema) os.environ['DBT_ENV_CUSTOM_ENV_env_key'] = 'env_value' @@ -181,159 +176,6 @@ def _no_stats(self): }, } - def _redshift_stats(self): - return { - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": True, - "description": "Indicates whether there are statistics for this table", - "include": False - }, - "encoded": { - "id": "encoded", - "label": "Encoded", - "value": AnyStringWith('Y'), - "description": "Indicates whether any column in the table has compression encoding defined.", - "include": True - }, - "diststyle": { - "id": "diststyle", - "label": "Dist Style", - "value": AnyStringWith('AUTO'), - "description": "Distribution style or distribution key column, if key distribution is defined.", - "include": True - }, - "max_varchar": { - "id": "max_varchar", - "label": "Max Varchar", - "value": AnyFloat(), - "description": "Size of the largest column that uses a VARCHAR data type.", - "include": True - }, - "size": { - "id": "size", - "label": "Approximate Size", - "value": AnyFloat(), - "description": "Approximate size of the table, calculated from a count of 1MB blocks", - "include": True - }, - 'sortkey1': { - 'id': 'sortkey1', - 'label': 'Sort Key 1', - 'value': AnyStringWith('AUTO'), - 'description': 'First column in the sort key.', - 'include': True, - }, - "pct_used": { - "id": "pct_used", - "label": "Disk Utilization", - "value": AnyFloat(), - "description": "Percent of available space that is used by the table.", - "include": True - }, - "stats_off": { - "id": "stats_off", - "label": "Stats Off", - "value": AnyFloat(), - "description": "Number that indicates how stale the table statistics are; 0 is current, 100 is out of date.", - "include": True - }, - "rows": { - "id": "rows", - "label": "Approximate Row Count", - "value": AnyFloat(), - "description": "Approximate number of rows in the table. This value includes rows marked for deletion, but not yet vacuumed.", - "include": True - }, - } - - def _snowflake_stats(self): - return { - 'has_stats': { - 'id': 'has_stats', - 'label': 'Has Stats?', - 'value': True, - 'description': 'Indicates whether there are statistics for this table', - 'include': False, - }, - 'bytes': { - 'id': 'bytes', - 'label': 'Approximate Size', - 'value': AnyFloat(), - 'description': 'Approximate size of the table as reported by Snowflake', - 'include': True, - }, - 'last_modified': { - 'id': 'last_modified', - 'label': 'Last Modified', - 'value': AnyString(), - 'description': 'The timestamp for last update/change', - 'include': True, - }, - 'row_count': { - 'id': 'row_count', - 'label': 'Row Count', - 'value': 1.0, - 'description': 'An approximate count of rows in this table', - 'include': True, - } - } - - def _bigquery_stats(self, is_table, partition=None, cluster=None): - stats = {} - - if is_table: - stats.update({ - 'num_bytes': { - 'id': 'num_bytes', - 'label': AnyString(), - 'value': AnyFloat(), - 'description': AnyString(), - 'include': True, - }, - 'num_rows': { - 'id': 'num_rows', - 'label': AnyString(), - 'value': AnyFloat(), - 'description': AnyString(), - 'include': True, - } - }) - - if partition is not None: - stats.update({ - 'partitioning_type': { - 'id': 'partitioning_type', - 'label': AnyString(), - 'value': partition, - 'description': AnyString(), - 'include': True - } - }) - - if cluster is not None: - stats.update({ - 'clustering_fields': { - 'id': 'clustering_fields', - 'label': AnyString(), - 'value': cluster, - 'description': AnyString(), - 'include': True - } - }) - - has_stats = { - 'id': 'has_stats', - 'label': 'Has Stats?', - 'value': bool(stats), - 'description': 'Indicates whether there are statistics for this table', - 'include': False, - } - stats['has_stats'] = has_stats - - return stats - def _expected_catalog(self, id_type, text_type, time_type, view_type, table_type, model_stats, seed_stats=None, case=None, case_columns=False, model_database=None): @@ -450,14 +292,11 @@ def expected_postgres_catalog(self): ) def get_role(self): - if self.adapter_type in {'postgres', 'redshift'}: + if self.adapter_type in {'postgres'}: profile = self.get_profile(self.adapter_type) target_name = profile['test']['target'] return profile['test']['outputs'][target_name]['user'] - elif self.adapter_type == 'snowflake': - return self.run_sql('select current_role()', fetch='one')[0] - else: # bigquery, presto, other dbs that have no 'role' - return None + return None def expected_postgres_references_catalog(self): model_database = self.default_database @@ -570,32 +409,6 @@ def expected_postgres_references_catalog(self): }, } - def expected_snowflake_catalog(self, case_columns=False): - return self._expected_catalog( - id_type='NUMBER', - text_type='TEXT', - time_type='TIMESTAMP_NTZ', - view_type='VIEW', - table_type='BASE TABLE', - model_stats=self._no_stats(), - seed_stats=self._snowflake_stats(), - case=lambda x: x.upper(), - model_database=self.alternative_database, - case_columns=case_columns, - ) - - def expected_bigquery_catalog(self): - return self._expected_catalog( - id_type='INT64', - text_type='STRING', - time_type='DATETIME', - view_type='view', - table_type='table', - model_stats=self._bigquery_stats(False), - seed_stats=self._bigquery_stats(True), - model_database=self.alternative_database, - ) - def expected_presto_catalog(self): return self._expected_catalog( id_type='integer', @@ -608,268 +421,6 @@ def expected_presto_catalog(self): model_database=self.default_database, ) - @staticmethod - def _clustered_bigquery_columns(update_type): - return { - 'id': { - 'comment': None, - 'index': 1, - 'name': 'id', - 'type': 'INT64', - }, - 'first_name': { - 'comment': None, - 'index': 2, - 'name': 'first_name', - 'type': 'STRING', - }, - 'email': { - 'comment': None, - 'index': 3, - 'name': 'email', - 'type': 'STRING', - }, - 'ip_address': { - 'comment': None, - 'index': 4, - 'name': 'ip_address', - 'type': 'STRING', - }, - 'updated_at': { - 'comment': None, - 'index': 5, - 'name': 'updated_at', - 'type': update_type, - }, - } - - def expected_bigquery_complex_catalog(self): - my_schema_name = self.unique_schema() - role = self.get_role() - table_stats = self._bigquery_stats(True) - clustering_stats = self._bigquery_stats(True, partition='updated_at', - cluster='first_name') - multi_clustering_stats = self._bigquery_stats(True, partition='updated_at', - cluster='first_name, email') - nesting_columns = { - 'field_1': { - 'name': 'field_1', - 'index': 1, - 'type': 'INT64', - 'comment': None - }, - 'field_2': { - 'name': 'field_2', - 'index': 2, - 'type': 'INT64', - 'comment': None - }, - 'field_3': { - 'name': 'field_3', - 'index': 3, - 'type': 'INT64', - 'comment': None - }, - 'nested_field': { - 'name': 'nested_field', - 'index': 4, - 'type': 'STRUCT', - 'comment': None - }, - 'nested_field.field_5': { - 'name': 'nested_field.field_5', - 'index': 5, - 'type': 'INT64', - 'comment': None - }, - 'nested_field.field_6': { - 'name': 'nested_field.field_6', - 'index': 6, - 'type': 'INT64', - 'comment': None - } - } - - return { - 'nodes': { - 'model.test.clustered': { - 'unique_id': 'model.test.clustered', - 'metadata': { - 'comment': None, - 'name': 'clustered', - 'owner': None, - 'schema': my_schema_name, - 'database': self.default_database, - 'type': 'table' - }, - 'stats': clustering_stats, - 'columns': self._clustered_bigquery_columns('DATE'), - }, - 'model.test.multi_clustered': { - 'unique_id': 'model.test.multi_clustered', - 'metadata': { - 'comment': None, - 'name': 'multi_clustered', - 'owner': None, - 'schema': my_schema_name, - 'database': self.default_database, - 'type': 'table' - }, - 'stats': multi_clustering_stats, - 'columns': self._clustered_bigquery_columns('DATE'), - }, - 'seed.test.seed': { - 'unique_id': 'seed.test.seed', - 'metadata': { - 'comment': None, - 'name': 'seed', - 'owner': None, - 'schema': my_schema_name, - 'database': self.default_database, - 'type': 'table', - }, - 'stats': table_stats, - 'columns': self._clustered_bigquery_columns('DATETIME'), - }, - 'model.test.nested_view': { - 'unique_id': 'model.test.nested_view', - 'metadata': { - 'schema': my_schema_name, - 'database': self.default_database, - 'name': 'nested_view', - 'type': 'view', - 'owner': role, - 'comment': None - }, - 'stats': self._bigquery_stats(False), - 'columns': nesting_columns, - }, - 'model.test.nested_table': { - 'unique_id': 'model.test.nested_table', - 'metadata': { - 'schema': my_schema_name, - 'database': self.default_database, - 'name': 'nested_table', - 'type': 'table', - 'owner': role, - 'comment': None - }, - 'stats': table_stats, - 'columns': nesting_columns, - }, - }, - 'sources': {}, - } - - def expected_redshift_catalog(self): - return self._expected_catalog( - id_type='integer', - text_type=AnyStringWith('character varying'), - time_type='timestamp without time zone', - view_type='VIEW', - table_type='BASE TABLE', - model_stats=self._no_stats(), - seed_stats=self._redshift_stats(), - ) - - def expected_redshift_incremental_catalog(self): - my_schema_name = self.unique_schema() - role = self.get_role() - return { - 'nodes': { - 'model.test.model': { - 'unique_id': 'model.test.model', - 'metadata': { - 'schema': my_schema_name, - 'database': self.default_database, - 'name': 'model', - 'type': 'LATE BINDING VIEW', - 'comment': None, - 'owner': role, - }, - # incremental views have no stats - 'stats': self._no_stats(), - 'columns': { - 'id': { - 'name': 'id', - 'index': 1, - 'type': 'integer', - 'comment': None, - }, - 'first_name': { - 'name': 'first_name', - 'index': 2, - 'type': 'character varying(5)', - 'comment': None, - }, - 'email': { - 'name': 'email', - 'index': 3, - 'type': 'character varying(23)', - 'comment': None, - }, - 'ip_address': { - 'name': 'ip_address', - 'index': 4, - 'type': 'character varying(14)', - 'comment': None, - }, - 'updated_at': { - 'name': 'updated_at', - 'index': 5, - 'type': 'timestamp without time zone', - 'comment': None, - }, - }, - }, - 'seed.test.seed': { - 'unique_id': 'seed.test.seed', - 'metadata': { - 'schema': my_schema_name, - 'database': self.default_database, - 'name': 'seed', - 'type': 'BASE TABLE', - 'comment': None, - 'owner': role, - }, - 'stats': self._redshift_stats(), - 'columns': { - 'id': { - 'name': 'id', - 'index': 1, - 'type': 'integer', - 'comment': None, - }, - 'first_name': { - 'name': 'first_name', - 'index': 2, - 'type': 'character varying(5)', - 'comment': None, - }, - 'email': { - 'name': 'email', - 'index': 3, - 'type': 'character varying(23)', - 'comment': None, - }, - 'ip_address': { - 'name': 'ip_address', - 'index': 4, - 'type': 'character varying(14)', - 'comment': None, - }, - 'updated_at': { - 'name': 'updated_at', - 'index': 5, - 'type': 'timestamp without time zone', - 'comment': None, - }, - }, - }, - }, - 'sources': {}, - } - def verify_catalog(self, expected): self.assertTrue(os.path.exists('./target/catalog.json')) @@ -1084,7 +635,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): target_schema=self.alternate_schema ) - quote_database = quote_schema = self.adapter_type != 'snowflake' + quote_database = quote_schema = True relation_name_node_format = self._relation_name_format( quote_database, quote_schema, quote_model ) @@ -2193,737 +1744,39 @@ def expected_postgres_references_manifest(self, model_database=None): } } - def expected_bigquery_complex_manifest(self): - nested_view_sql_path = self.dir('bq_models/nested_view.sql') - nested_table_sql_path = self.dir('bq_models/nested_table.sql') - clustered_sql_path = self.dir('bq_models/clustered.sql') - multi_clustered_sql_path = self.dir('bq_models/multi_clustered.sql') - seed_path = self.dir('seed/seed.csv') - snapshot_path = self.dir('snapshot/snapshot_seed.sql') - my_schema_name = self.unique_schema() + def _checksum_file(self, path): + """windows has silly git behavior that adds newlines, and python does + silly things if we just open(..., 'r').encode('utf-8'). + """ + with open(self.dir(path), 'rb') as fp: + hashed = hashlib.sha256(fp.read()).hexdigest() + return { + 'name': 'sha256', + 'checksum': hashed, + } + def _path_to(self, searched_path: str, relative_path: str): return { - 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v3.json', - 'dbt_version': dbt.version.__version__, - 'nodes': { - 'model.test.clustered': { - 'alias': 'clustered', - 'config': self.rendered_model_config( - cluster_by=['first_name'], - materialized='table', - partition_by={'field': 'updated_at', - 'data_type': 'date'}, - ), - 'sources': [], - 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, - 'fqn': ['test', 'clustered'], - 'compiled_path': Normalized('target/compiled/test/bq_models/clustered.sql'), - 'build_path': None, - 'created_at': ANY, - 'name': 'clustered', - 'original_file_path': clustered_sql_path, - 'package_name': 'test', - 'path': 'clustered.sql', - 'raw_sql': LineIndifferent(_read_file(clustered_sql_path).rstrip('\r\n')), - 'refs': [['seed']], - 'relation_name': '`{0}`.`{1}`.clustered'.format( - self.default_database, my_schema_name - ), - 'resource_type': 'model', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'model.test.clustered', - 'columns': { - 'email': { - 'description': "The user's email", - 'name': 'email', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'description': "The user's name", - 'name': 'first_name', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'id': { - 'description': 'The user id', - 'name': 'id', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'description': "The user's IP address", - 'name': 'ip_address', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'description': 'When the user was updated', - 'name': 'updated_at', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'deferred': False, - 'description': 'A clustered and partitioned copy of the test model', - 'patch_path': 'test://' + self.dir('bq_models/schema.yml'), - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(clustered_sql_path), - 'unrendered_config': self.unrendered_model_config( - cluster_by=['first_name'], - materialized='table', - partition_by={'field': 'updated_at', - 'data_type': 'date'}, - ), - }, - 'model.test.multi_clustered': { - 'alias': 'multi_clustered', - 'compiled_path': Normalized('target/compiled/test/bq_models/multi_clustered.sql'), - 'build_path': None, - 'created_at': ANY, - 'config': self.rendered_model_config( - cluster_by=['first_name', 'email'], - materialized='table', - partition_by={'field': 'updated_at', - 'data_type': 'date'} - ), - 'sources': [], - 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, - 'fqn': ['test', 'multi_clustered'], - 'name': 'multi_clustered', - 'original_file_path': multi_clustered_sql_path, - 'package_name': 'test', - 'path': 'multi_clustered.sql', - 'raw_sql': LineIndifferent(_read_file(multi_clustered_sql_path).rstrip('\r\n')), - 'refs': [['seed']], - 'relation_name': '`{0}`.`{1}`.multi_clustered'.format( - self.default_database, my_schema_name - ), - 'resource_type': 'model', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'model.test.multi_clustered', - 'columns': { - 'email': { - 'description': "The user's email", - 'name': 'email', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'description': "The user's name", - 'name': 'first_name', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'id': { - 'description': 'The user id', - 'name': 'id', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'description': "The user's IP address", - 'name': 'ip_address', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'description': 'When the user was updated', - 'name': 'updated_at', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'deferred': False, - 'description': 'A clustered and partitioned copy of the test model, clustered on multiple columns', - 'patch_path': 'test://' + self.dir('bq_models/schema.yml'), - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(multi_clustered_sql_path), - 'unrendered_config': self.unrendered_model_config( - cluster_by=['first_name', 'email'], - materialized='table', - partition_by={'field': 'updated_at', - 'data_type': 'date'} - ), - }, - 'model.test.nested_view': { - 'alias': 'nested_view', - 'compiled_path': Normalized('target/compiled/test/bq_models/nested_view.sql'), - 'build_path': None, - 'created_at': ANY, - 'config': self.rendered_model_config(), - 'sources': [], - 'depends_on': { - 'macros': [], - 'nodes': ['model.test.nested_table'] - }, - 'fqn': ['test', 'nested_view'], - 'name': 'nested_view', - 'original_file_path': nested_view_sql_path, - 'package_name': 'test', - 'path': 'nested_view.sql', - 'raw_sql': LineIndifferent(_read_file(nested_view_sql_path).rstrip('\r\n')), - 'refs': [['nested_table']], - 'relation_name': '`{0}`.`{1}`.nested_view'.format( - self.default_database, my_schema_name - ), - 'resource_type': 'model', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'model.test.nested_view', - 'columns': { - 'field_1': { - 'name': 'field_1', - 'description': 'The first field', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'field_2': { - 'name': 'field_2', - 'description': 'The second field', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'field_3': { - 'name': 'field_3', - 'description': 'The third field', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'nested_field.field_4': { - 'name': 'nested_field.field_4', - 'description': 'The first nested field', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'nested_field.field_5': { - 'name': 'nested_field.field_5', - 'description': 'The second nested field', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'deferred': False, - 'description': 'The test model', - 'patch_path': 'test://' + self.dir('bq_models/schema.yml'), - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(nested_view_sql_path), - 'unrendered_config': self.unrendered_model_config(materialized='view'), - }, - 'model.test.nested_table': { - 'alias': 'nested_table', - 'compiled_path': Normalized('target/compiled/test/bq_models/nested_table.sql'), - 'build_path': None, - 'created_at': ANY, - 'config': self.rendered_model_config(materialized='table'), - 'sources': [], - 'depends_on': { - 'macros': [], - 'nodes': [] - }, - 'fqn': ['test', 'nested_table'], - 'name': 'nested_table', - 'original_file_path': nested_table_sql_path, - 'package_name': 'test', - 'patch_path': None, - 'path': 'nested_table.sql', - 'raw_sql': LineIndifferent(_read_file(nested_table_sql_path).rstrip('\r\n')), - 'refs': [], - 'relation_name': '`{0}`.`{1}`.nested_table'.format( - self.default_database, my_schema_name - ), - 'resource_type': 'model', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'model.test.nested_table', - 'columns': {}, - 'deferred': False, - 'description': '', - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(nested_table_sql_path), - 'unrendered_config': self.unrendered_model_config(materialized='table'), - }, - 'seed.test.seed': { - 'compiled_path': None, - 'build_path': None, - 'created_at': ANY, - 'patch_path': 'test://' + self.dir('seed/schema.yml'), - 'path': 'seed.csv', - 'name': 'seed', - 'root_path': self.test_root_realpath, - 'relation_name': '`{0}`.`{1}`.seed'.format( - self.default_database, my_schema_name - ), - 'resource_type': 'seed', - 'raw_sql': '', - 'package_name': 'test', - 'original_file_path': seed_path, - 'refs': [], - 'sources': [], - 'depends_on': { - 'nodes': [], - 'macros': [], - }, - 'unique_id': 'seed.test.seed', - 'fqn': ['test', 'seed'], - 'tags': [], - 'meta': {}, - 'config': self.rendered_seed_config(), - 'schema': my_schema_name, - 'database': self.default_database, - 'alias': 'seed', - 'columns': { - 'id': { - 'name': 'id', - 'description': 'The user ID number', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'name': 'first_name', - 'description': "The user's first name", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'email': { - 'name': 'email', - 'description': "The user's email", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'name': 'ip_address', - 'description': "The user's IP address", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'name': 'updated_at', - 'description': "The last time this user's email was updated", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'deferred': False, - 'description': 'The test seed', - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': '', - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(seed_path), - 'unrendered_config': self.unrendered_seed_config(), - }, - 'snapshot.test.snapshot_seed': { - 'alias': 'snapshot_seed', - 'compiled_path': None, - 'build_path': None, - 'created_at': ANY, - 'checksum': self._checksum_file(snapshot_path), - 'columns': {}, - 'compiled': True, - 'compiled_sql': ANY, - 'config': self.rendered_snapshot_config( - target_schema=self.alternate_schema - ), - 'database': self.default_database, - 'deferred': False, - 'depends_on': {'macros': [], - 'nodes': ['seed.test.seed']}, - 'description': '', - 'docs': {'show': True}, - 'extra_ctes': [], - 'extra_ctes_injected': True, - 'fqn': ['test', 'snapshot_seed', 'snapshot_seed'], - 'meta': {}, - 'name': 'snapshot_seed', - 'original_file_path': snapshot_path, - 'package_name': 'test', - 'patch_path': None, - 'path': 'snapshot_seed.sql', - 'raw_sql': ANY, - 'refs': [['seed']], - 'relation_name': '`{0}`.`{1}`.snapshot_seed'.format( - self.default_database, self.alternate_schema - ), - 'resource_type': 'snapshot', - 'root_path': self.test_root_realpath, - 'schema': self.alternate_schema, - 'sources': [], - 'tags': [], - 'unique_id': 'snapshot.test.snapshot_seed', - 'unrendered_config': self.unrendered_snapshot_config( - target_schema=self.alternate_schema - )} - }, - 'sources': {}, - 'exposures': {}, - 'selectors': {}, - 'child_map': { - 'model.test.clustered': [], - 'model.test.multi_clustered': [], - 'model.test.nested_table': ['model.test.nested_view'], - 'model.test.nested_view': [], - 'seed.test.seed': ['model.test.clustered', - 'model.test.multi_clustered', - 'snapshot.test.snapshot_seed'], - 'snapshot.test.snapshot_seed': [] - }, - 'parent_map': { - 'model.test.clustered': ['seed.test.seed'], - 'model.test.multi_clustered': ['seed.test.seed'], - 'seed.test.seed': [], - 'snapshot.test.snapshot_seed': ['seed.test.seed'], - 'model.test.nested_table': [], - 'model.test.nested_view': ['model.test.nested_table'], - }, - 'docs': { - 'dbt.__overview__': ANY, - 'test.macro_info': ANY, - 'test.macro_arg_info': ANY, - }, - 'disabled': {}, - } - - def _checksum_file(self, path): - """windows has silly git behavior that adds newlines, and python does - silly things if we just open(..., 'r').encode('utf-8'). - """ - with open(self.dir(path), 'rb') as fp: - hashed = hashlib.sha256(fp.read()).hexdigest() - return { - 'name': 'sha256', - 'checksum': hashed, - } - - def _path_to(self, searched_path: str, relative_path: str): - return { - 'searched_path': normalize(searched_path), - 'relative_path': normalize(relative_path), - 'project_root': normalize(self.test_root_dir), - } - - def _absolute_path_to(self, searched_path: str, relative_path: str): - return os.path.join( - normalize(self.test_root_dir), - normalize(searched_path), - normalize(relative_path) - ) - - def _relation_name_format(self, quote_database: bool, quote_schema: bool, - quote_identifier: bool): - return ".".join(( - self._quote("{0}") if quote_database else '{0}', - self._quote("{1}") if quote_schema else '{1}', - self._quote("{2}") if quote_identifier else '{2}', - )) - - def expected_redshift_incremental_view_manifest(self): - model_sql_path = self.dir('rs_models/model.sql') - my_schema_name = self.unique_schema() - seed_path = self.dir('seed/seed.csv') - snapshot_path = self.dir('snapshot/snapshot_seed.sql') - - return { - 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v3.json', - 'dbt_version': dbt.version.__version__, - 'nodes': { - 'model.test.model': { - 'compiled_path': Normalized('target/compiled/test/rs_models/model.sql'), - 'build_path': None, - 'created_at': ANY, - 'name': 'model', - 'root_path': self.test_root_realpath, - 'relation_name': '"{0}"."{1}".model'.format( - self.default_database, my_schema_name - ), - 'resource_type': 'model', - 'path': 'model.sql', - 'original_file_path': model_sql_path, - 'package_name': 'test', - 'raw_sql': LineIndifferent(_read_file(model_sql_path).rstrip('\r\n')), - 'refs': [['seed']], - 'sources': [], - 'depends_on': { - 'nodes': ['seed.test.seed'], - 'macros': [], - }, - 'unique_id': 'model.test.model', - 'fqn': ['test', 'model'], - 'tags': [], - 'meta': {}, - 'config': self.rendered_model_config(bind=False), - 'schema': my_schema_name, - 'database': self.default_database, - 'alias': 'model', - 'deferred': False, - 'description': 'The test model', - 'columns': { - 'id': { - 'name': 'id', - 'description': 'The user ID number', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'name': 'first_name', - 'description': "The user's first name", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'email': { - 'name': 'email', - 'description': "The user's email", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'name': 'ip_address', - 'description': "The user's IP address", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'name': 'updated_at', - 'description': "The last time this user's email was updated", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'patch_path': 'test://' + self.dir('rs_models/schema.yml'), - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(model_sql_path), - 'unrendered_config': self.unrendered_model_config(bind=False, materialized='view'), - }, - 'seed.test.seed': { - 'compiled_path': None, - 'build_path': None, - 'created_at': ANY, - 'patch_path': 'test://' + self.dir('seed/schema.yml'), - 'path': 'seed.csv', - 'name': 'seed', - 'root_path': self.test_root_realpath, - 'relation_name': '"{0}"."{1}".seed'.format( - self.default_database, my_schema_name - ), - 'resource_type': 'seed', - 'raw_sql': '', - 'package_name': 'test', - 'original_file_path': seed_path, - 'refs': [], - 'sources': [], - 'depends_on': { - 'nodes': [], - 'macros': [], - }, - 'unique_id': 'seed.test.seed', - 'fqn': ['test', 'seed'], - 'tags': [], - 'meta': {}, - 'config': self.rendered_seed_config(), - 'schema': my_schema_name, - 'database': self.default_database, - 'alias': 'seed', - 'columns': { - 'id': { - 'name': 'id', - 'description': 'The user ID number', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'name': 'first_name', - 'description': "The user's first name", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'email': { - 'name': 'email', - 'description': "The user's email", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'name': 'ip_address', - 'description': "The user's IP address", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'name': 'updated_at', - 'description': "The last time this user's email was updated", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'deferred': False, - 'description': 'The test seed', - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(seed_path), - 'unrendered_config': self.unrendered_seed_config(), - }, - 'snapshot.test.snapshot_seed': { - 'alias': 'snapshot_seed', - 'compiled_path': None, - 'build_path': None, - 'created_at': ANY, - 'checksum': self._checksum_file(snapshot_path), - 'columns': {}, - 'compiled': True, - 'compiled_sql': ANY, - 'config': self.rendered_snapshot_config( - target_schema=self.alternate_schema - ), - 'database': self.default_database, - 'deferred': False, - 'depends_on': {'macros': [], - 'nodes': ['seed.test.seed']}, - 'description': '', - 'docs': {'show': True}, - 'extra_ctes': [], - 'extra_ctes_injected': True, - 'fqn': ['test', 'snapshot_seed', 'snapshot_seed'], - 'meta': {}, - 'name': 'snapshot_seed', - 'original_file_path': snapshot_path, - 'package_name': 'test', - 'patch_path': None, - 'path': 'snapshot_seed.sql', - 'raw_sql': ANY, - 'refs': [['seed']], - 'relation_name': '"{0}"."{1}".snapshot_seed'.format( - self.default_database, self.alternate_schema - ), - 'resource_type': 'snapshot', - 'root_path': self.test_root_realpath, - 'schema': self.alternate_schema, - 'sources': [], - 'tags': [], - 'unique_id': 'snapshot.test.snapshot_seed', - 'unrendered_config': self.unrendered_snapshot_config( - target_schema=self.alternate_schema - )} - }, - 'sources': {}, - 'exposures': {}, - 'selectors': {}, - 'parent_map': { - 'model.test.model': ['seed.test.seed'], - 'seed.test.seed': [], - 'snapshot.test.snapshot_seed': ['seed.test.seed'] - }, - 'child_map': { - 'model.test.model': [], - 'seed.test.seed': ['model.test.model', - 'snapshot.test.snapshot_seed'], - 'snapshot.test.snapshot_seed': [] - }, - 'docs': { - 'dbt.__overview__': ANY, - 'test.macro_info': ANY, - 'test.macro_arg_info': ANY, - }, - 'disabled': {}, + 'searched_path': normalize(searched_path), + 'relative_path': normalize(relative_path), + 'project_root': normalize(self.test_root_dir), } + def _absolute_path_to(self, searched_path: str, relative_path: str): + return os.path.join( + normalize(self.test_root_dir), + normalize(searched_path), + normalize(relative_path) + ) + + def _relation_name_format(self, quote_database: bool, quote_schema: bool, + quote_identifier: bool): + return ".".join(( + self._quote("{0}") if quote_database else '{0}', + self._quote("{1}") if quote_schema else '{1}', + self._quote("{2}") if quote_identifier else '{2}', + )) + def verify_metadata(self, metadata, dbt_schema_version): assert 'generated_at' in metadata self.assertBetween(metadata['generated_at'], @@ -2970,7 +1823,7 @@ def verify_manifest(self, expected_manifest): self.assertEqual(manifest[key], expected_manifest[key]) def _quote(self, value): - quote_char = '`' if self.adapter_type == 'bigquery' else '"' + quote_char = '"' return '{0}{1}{0}'.format(quote_char, value) def expected_run_results(self): @@ -3155,115 +2008,6 @@ def test_postgres_asset_paths_copied(self): assert not os.path.exists('./target/non-existent-assets') - @use_profile('snowflake') - def test__snowflake__run_and_generate(self): - self.run_and_generate() - - self.verify_catalog(self.expected_snowflake_catalog()) - self.verify_manifest(self.expected_seeded_manifest()) - self.verify_run_results(self.expected_run_results()) - - @use_profile('snowflake') - def test__snowflake__run_and_generate_ignore_quoting_parameter(self): - # with optional adapters, this package could easily just not exist! - # accordingly, only run it when we think snowflake things should work - from dbt.adapters.snowflake import connections as snowflake_conn - old_connect = snowflake_conn.snowflake.connector.connect - - def connect(*args, **kwargs): - kwargs['session_parameters'] = { - 'QUOTED_IDENTIFIERS_IGNORE_CASE': True - } - return old_connect(*args, **kwargs) - - with patch.object(snowflake_conn.snowflake.connector, 'connect', connect): - self.run_and_generate({ - 'quoting': { - 'identifier': True, - } - }) - - self.verify_catalog(self.expected_snowflake_catalog(case_columns=True)) - self.verify_manifest(self.expected_seeded_manifest(quote_model=True)) - self.verify_run_results(self.expected_run_results()) - - @use_profile('bigquery') - def test__bigquery__run_and_generate(self): - self.run_and_generate() - - self.verify_catalog(self.expected_bigquery_catalog()) - self.verify_manifest(self.expected_seeded_manifest()) - self.verify_run_results(self.expected_run_results()) - - @use_profile('bigquery') - def test__bigquery__complex_models(self): - self.run_and_generate( - extra={'source-paths': [self.dir('bq_models')]}, - model_count=4 - ) - - self.verify_catalog(self.expected_bigquery_complex_catalog()) - self.verify_manifest(self.expected_bigquery_complex_manifest()) - - @use_profile('redshift') - def test__redshift__run_and_generate(self): - self.run_and_generate(alternate_db=self.default_database) - self.verify_catalog(self.expected_redshift_catalog()) - self.verify_manifest(self.expected_seeded_manifest( - model_database=self.default_database - )) - self.verify_run_results(self.expected_run_results()) - - @use_profile('redshift') - def test__redshift__incremental_view(self): - self.run_and_generate( - {'source-paths': [self.dir('rs_models')]}, - alternate_db=self.default_database, - model_count=1, - ) - self.verify_catalog(self.expected_redshift_incremental_catalog()) - self.verify_manifest( - self.expected_redshift_incremental_view_manifest()) - - @use_profile('presto') - def test__presto__run_and_generate(self): - self.run_and_generate(alternate_db=self.default_database) - self.verify_catalog(self.expected_presto_catalog()) - self.verify_manifest(self.expected_seeded_manifest( - model_database=self.default_database - )) - self.verify_run_results(self.expected_run_results()) - - -class TestDocsGenerateMissingSchema(DBTIntegrationTest): - @property - def schema(self): - return 'docs_generate_029' - - @staticmethod - def dir(path): - return normalize(path) - - @property - def models(self): - return self.dir("bq_models_noschema") - - def setUp(self): - super().setUp() - self.extra_schema = self.unique_schema() + '_bq_test' - - def tearDown(self): - with self.adapter.connection_named('__test'): - self._drop_schema_named(self.default_database, self.extra_schema) - super().tearDown() - - @use_profile('bigquery') - def test_bigquery_docs_generate_noschema(self): - self.run_dbt([ - 'docs', 'generate', - '--vars', "{{extra_schema: {}}}".format(self.extra_schema) - ]) - class TestDocsGenerateOverride(DBTIntegrationTest): diff --git a/test/integration/030_statement_test/models-bq/statement_actual.sql b/test/integration/030_statement_test/models-bq/statement_actual.sql deleted file mode 100644 index 92f9ab1ab95..00000000000 --- a/test/integration/030_statement_test/models-bq/statement_actual.sql +++ /dev/null @@ -1,23 +0,0 @@ - --- {{ ref('seed') }} - -{%- call statement('test_statement', fetch_result=True) -%} - - select - count(*) as `num_records` - - from {{ ref('seed') }} - -{%- endcall -%} - -{% set result = load_result('test_statement') %} - -{% set res_table = result['table'] %} -{% set res_matrix = result['data'] %} - -{% set matrix_value = res_matrix[0][0] %} -{% set table_value = res_table[0]['num_records'] %} - -select 'matrix' as source, {{ matrix_value }} as value -union all -select 'table' as source, {{ table_value }} as value diff --git a/test/integration/030_statement_test/snowflake-seed/seed.csv b/test/integration/030_statement_test/snowflake-seed/seed.csv deleted file mode 100644 index 45d6159f806..00000000000 --- a/test/integration/030_statement_test/snowflake-seed/seed.csv +++ /dev/null @@ -1,101 +0,0 @@ -ID,FIRST_NAME,LAST_NAME,EMAIL,GENDER,IP_ADDRESS -1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 -2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 -3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 -4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 -5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 -6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 -7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 -8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 -9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 -10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 -11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 -12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 -13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 -14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 -15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 -16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 -17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 -18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 -19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 -20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 -21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 -22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 -23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 -24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 -25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 -26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 -27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 -28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 -29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 -30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 -31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 -32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 -33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 -34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 -35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 -36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 -37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 -38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 -39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 -40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 -41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 -42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 -43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 -44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 -45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 -46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 -47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 -48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 -49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 -50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 -51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 -52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 -53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 -54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 -55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 -56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 -57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 -58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 -59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 -60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 -61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 -62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 -63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 -64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 -65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 -66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 -67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 -68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 -69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 -70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 -71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 -72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 -73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 -74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 -75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 -76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 -77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 -78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 -79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 -80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 -81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 -82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 -83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 -84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 -85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 -86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 -87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 -88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 -89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 -90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 -91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 -92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 -93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 -94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 -95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 -96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 -97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 -98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 -99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 -100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 diff --git a/test/integration/030_statement_test/snowflake-seed/statement_expected.csv b/test/integration/030_statement_test/snowflake-seed/statement_expected.csv deleted file mode 100644 index e75249c094a..00000000000 --- a/test/integration/030_statement_test/snowflake-seed/statement_expected.csv +++ /dev/null @@ -1,3 +0,0 @@ -SOURCE,VALUE -matrix,100 -table,100 diff --git a/test/integration/030_statement_test/test_statements.py b/test/integration/030_statement_test/test_statements.py index a410cd86ab2..d441552051a 100644 --- a/test/integration/030_statement_test/test_statements.py +++ b/test/integration/030_statement_test/test_statements.py @@ -35,17 +35,6 @@ def test_postgres_statements(self): self.assertTablesEqual("statement_actual", "statement_expected") - @use_profile("snowflake") - def test_snowflake_statements(self): - self.use_default_project({"data-paths": [self.dir("snowflake-seed")]}) - - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 2) - results = self.run_dbt() - self.assertEqual(len(results), 1) - - self.assertManyTablesEqual(["STATEMENT_ACTUAL", "STATEMENT_EXPECTED"]) - @use_profile("presto") def test_presto_statements(self): self.use_default_project({"data-paths": [self.dir("seed")]}) @@ -57,37 +46,3 @@ def test_presto_statements(self): self.assertTablesEqual("statement_actual", "statement_expected") - -class TestStatementsBigquery(DBTIntegrationTest): - - @property - def schema(self): - return "statements_030" - - @staticmethod - def dir(path): - return path.lstrip("/") - - @property - def models(self): - return self.dir("models-bq") - - @property - def project_config(self): - return { - 'config-version': 2, - 'seeds': { - 'quote_columns': False, - } - } - - @use_profile("bigquery") - def test_bigquery_statements(self): - self.use_default_project({"data-paths": [self.dir("seed")]}) - - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 2) - results = self.run_dbt() - self.assertEqual(len(results), 1) - - self.assertTablesEqual("statement_actual", "statement_expected") diff --git a/test/integration/032_concurrent_transaction_test/README.md b/test/integration/032_concurrent_transaction_test/README.md deleted file mode 100644 index 48ece86154b..00000000000 --- a/test/integration/032_concurrent_transaction_test/README.md +++ /dev/null @@ -1,33 +0,0 @@ - -This test warrants some explanation. In dbt <=0.10.1, Redshift table and view materializations suffered from issues around concurrent transactions. In order to reliably reproduce this error, a query needs to select from a dbt model as the table is being rebuilt. Critically, this concurrent select needs to query the table during the drop/swap portition of the materialization. This looks like: - -```sql -begin; -create table as (...); -drop table old_table cascade; -// <---- The concurrent query needs to be running here! -alter table new_table rename to old_table; -commit; -``` - -In order to reliably reproduce this failure, the model shown above needs to block for a long time between the `drop` and `alter` statements. We can't just stick a sleep() call in there, as this code is defined in the materialization. Instead, we can reliably reproduce the failure by: - -1) creating a view that depends on this model -2) issuing a long-running query on the view before `dbt run` is invoked -3) issuing _another_ long-running query against the original model - -Since long-running query (step 2) is selecting from the view, Redshift blocks on the `drop ... cascade`, of the materialization, which causes the query from step 3 time to overlap with the critical section of the materialization between the `drop` and `alter` statements. - -In dbt v0.10.1, this integration test results in: - -``` -====================================================================== -FAIL: test__redshift__concurrent_transaction (test_concurrent_transaction.TestConcurrentTransaction) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/usr/src/app/test/integration/032_concurrent_transaction_test/test_concurrent_transaction.py", line 84, in test__redshift__concurrent_transaction - self.assertEqual(self.query_state['model_1'], 'good') -AssertionError: 'error: table 3379442 dropped by concurrent transaction\n' != 'good' -- error: table 3379442 dropped by concurrent transaction -+ good -``` diff --git a/test/integration/032_concurrent_transaction_test/macros/udfs.sql b/test/integration/032_concurrent_transaction_test/macros/udfs.sql deleted file mode 100644 index 8fc46d110da..00000000000 --- a/test/integration/032_concurrent_transaction_test/macros/udfs.sql +++ /dev/null @@ -1,13 +0,0 @@ - -{% macro create_udfs() %} - -CREATE OR REPLACE FUNCTION {{ target.schema }}.f_sleep (x float) -RETURNS bool IMMUTABLE -AS -$$ - from time import sleep - sleep(x) - return True -$$ LANGUAGE plpythonu; - -{% endmacro %} diff --git a/test/integration/032_concurrent_transaction_test/models-incremental/model_1.sql b/test/integration/032_concurrent_transaction_test/models-incremental/model_1.sql deleted file mode 100644 index 3d8ac43bae5..00000000000 --- a/test/integration/032_concurrent_transaction_test/models-incremental/model_1.sql +++ /dev/null @@ -1,9 +0,0 @@ - -{{ config(materialized='incremental', unique_key='id') }} - --- incremental model -select 1 as id - -{% if is_incremental() %} - where TRUE -{% endif %} diff --git a/test/integration/032_concurrent_transaction_test/models-incremental/view_model.sql b/test/integration/032_concurrent_transaction_test/models-incremental/view_model.sql deleted file mode 100644 index 40b85c8fc16..00000000000 --- a/test/integration/032_concurrent_transaction_test/models-incremental/view_model.sql +++ /dev/null @@ -1,3 +0,0 @@ - - -select * from {{ ref('model_1') }} diff --git a/test/integration/032_concurrent_transaction_test/models-table/model_1.sql b/test/integration/032_concurrent_transaction_test/models-table/model_1.sql deleted file mode 100644 index 344a2e8bd0f..00000000000 --- a/test/integration/032_concurrent_transaction_test/models-table/model_1.sql +++ /dev/null @@ -1,5 +0,0 @@ - -{{ config(materialized='table') }} - --- table model -select 1 as id diff --git a/test/integration/032_concurrent_transaction_test/models-table/view_model.sql b/test/integration/032_concurrent_transaction_test/models-table/view_model.sql deleted file mode 100644 index 40b85c8fc16..00000000000 --- a/test/integration/032_concurrent_transaction_test/models-table/view_model.sql +++ /dev/null @@ -1,3 +0,0 @@ - - -select * from {{ ref('model_1') }} diff --git a/test/integration/032_concurrent_transaction_test/models-view/model_1.sql b/test/integration/032_concurrent_transaction_test/models-view/model_1.sql deleted file mode 100644 index 21a96e98c79..00000000000 --- a/test/integration/032_concurrent_transaction_test/models-view/model_1.sql +++ /dev/null @@ -1,5 +0,0 @@ - -{{ config(materialized='view') }} - --- view model -select 1 as id diff --git a/test/integration/032_concurrent_transaction_test/models-view/view_model.sql b/test/integration/032_concurrent_transaction_test/models-view/view_model.sql deleted file mode 100644 index 40b85c8fc16..00000000000 --- a/test/integration/032_concurrent_transaction_test/models-view/view_model.sql +++ /dev/null @@ -1,3 +0,0 @@ - - -select * from {{ ref('model_1') }} diff --git a/test/integration/032_concurrent_transaction_test/test_concurrent_transaction.py b/test/integration/032_concurrent_transaction_test/test_concurrent_transaction.py deleted file mode 100644 index c1051bb584b..00000000000 --- a/test/integration/032_concurrent_transaction_test/test_concurrent_transaction.py +++ /dev/null @@ -1,136 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile -import threading -from dbt.adapters.factory import FACTORY - - -def get_adapter_standalone(config): - plugin = FACTORY.plugins[config.credentials.type] - cls = plugin.adapter - return cls(config) - - -class BaseTestConcurrentTransaction(DBTIntegrationTest): - - def reset(self): - self.query_state = { - 'view_model': 'wait', - 'model_1': 'wait', - } - - def setUp(self): - super().setUp() - self._secret_adapter = get_adapter_standalone(self.config) - self.reset() - - def tearDown(self): - self._secret_adapter.cleanup_connections() - super().tearDown() - - @property - def schema(self): - return "concurrent_transaction_032" - - @property - def project_config(self): - return { - 'config-version': 2, - "macro-paths": ["macros"], - "on-run-start": [ - "{{ create_udfs() }}", - ], - } - - def run_select_and_check(self, rel, sql): - connection_name = '__test_{}'.format(id(threading.current_thread())) - try: - with self._secret_adapter.connection_named(connection_name): - conn = self._secret_adapter.connections.get_thread_connection() - res = self.run_sql_common(self.transform_sql(sql), 'one', conn) - - # The result is the output of f_sleep(), which is True - if res[0]: - self.query_state[rel] = 'good' - else: - self.query_state[rel] = 'bad' - - except Exception as e: - if 'concurrent transaction' in str(e): - self.query_state[rel] = 'error: {}'.format(e) - else: - self.query_state[rel] = 'error: {}'.format(e) - - def async_select(self, rel, sleep=10): - # Run the select statement in a thread. When the query returns, the global - # query_state will be update with a state of good/bad/error, and the associated - # error will be reported if one was raised. - - schema = self.unique_schema() - query = ''' - -- async_select: {rel} - select {schema}.f_sleep({sleep}) from {schema}.{rel} - '''.format( - schema=schema, - sleep=sleep, - rel=rel) - - thread = threading.Thread(target=self.run_select_and_check, args=(rel, query)) - thread.start() - return thread - - def run_test(self): - self.use_profile("redshift") - - # First run the project to make sure the models exist - results = self.run_dbt(args=['run']) - self.assertEqual(len(results), 2) - - # Execute long-running queries in threads - t1 = self.async_select('view_model', 10) - t2 = self.async_select('model_1', 5) - - # While the queries are executing, re-run the project - res = self.run_dbt(args=['run', '--threads', '8']) - self.assertEqual(len(res), 2) - - # Finally, wait for these threads to finish - t1.join() - t2.join() - - self.assertTrue(len(res) > 0) - - # If the query succeeded, the global query_state should be 'good' - self.assertEqual(self.query_state['view_model'], 'good') - self.assertEqual(self.query_state['model_1'], 'good') - - -class TableTestConcurrentTransaction(BaseTestConcurrentTransaction): - @property - def models(self): - return "models-table" - - @use_profile("redshift") - def test__redshift__concurrent_transaction_table(self): - self.reset() - self.run_test() - - -class ViewTestConcurrentTransaction(BaseTestConcurrentTransaction): - @property - def models(self): - return "models-view" - - @use_profile("redshift") - def test__redshift__concurrent_transaction_view(self): - self.reset() - self.run_test() - - -class IncrementalTestConcurrentTransaction(BaseTestConcurrentTransaction): - @property - def models(self): - return "models-incremental" - - @use_profile("redshift") - def test__redshift__concurrent_transaction_incremental(self): - self.reset() - self.run_test() diff --git a/test/integration/034_redshift_test/models/model.sql b/test/integration/034_redshift_test/models/model.sql deleted file mode 100644 index 8ca73c7a0f4..00000000000 --- a/test/integration/034_redshift_test/models/model.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ - config( - materialized='view', bind=False - ) -}} - -select * from {{ ref('seed') }} diff --git a/test/integration/034_redshift_test/seed/seed.csv b/test/integration/034_redshift_test/seed/seed.csv deleted file mode 100644 index ef154f552c9..00000000000 --- a/test/integration/034_redshift_test/seed/seed.csv +++ /dev/null @@ -1,2 +0,0 @@ -id,first_name,email,ip_address,updated_at -1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31 diff --git a/test/integration/034_redshift_test/test_late_binding_view.py b/test/integration/034_redshift_test/test_late_binding_view.py deleted file mode 100644 index 5f7816e00e9..00000000000 --- a/test/integration/034_redshift_test/test_late_binding_view.py +++ /dev/null @@ -1,39 +0,0 @@ -import os - -from test.integration.base import DBTIntegrationTest, use_profile - - -class TestLateBindingView(DBTIntegrationTest): - @property - def schema(self): - return 'late_binding_view_034' - - @staticmethod - def dir(path): - return os.path.normpath(path) - - @property - def models(self): - return self.dir("models") - - @property - def project_config(self): - return { - 'config-version': 2, - 'data-paths': [self.dir('seed')], - 'seeds': { - 'quote_columns': False, - } - } - - @use_profile('redshift') - def test__redshift_late_binding_view_query(self): - self.assertEqual(len(self.run_dbt(["seed"])), 1) - self.assertEqual(len(self.run_dbt()), 1) - # remove the table. Use 'cascade' here so that if late-binding views - # didn't work as advertised, the following dbt run will fail. - drop = 'drop table if exists {}.seed cascade'.format( - self.unique_schema() - ) - self.run_sql(drop) - self.assertEqual(len(self.run_dbt()), 1) diff --git a/test/integration/035_changing_relation_type_test/test_changing_relation_type.py b/test/integration/035_changing_relation_type_test/test_changing_relation_type.py index a10a1f424c3..f1d31b98601 100644 --- a/test/integration/035_changing_relation_type_test/test_changing_relation_type.py +++ b/test/integration/035_changing_relation_type_test/test_changing_relation_type.py @@ -1,5 +1,5 @@ from pytest import mark -from test.integration.base import DBTIntegrationTest, use_profile, bigquery_rate_limiter +from test.integration.base import DBTIntegrationTest, use_profile class TestChangingRelationType(DBTIntegrationTest): @@ -43,55 +43,3 @@ def swap_types_and_test(self): @use_profile("postgres") def test__postgres__switch_materialization(self): self.swap_types_and_test() - - @use_profile("snowflake") - def test__snowflake__switch_materialization(self): - self.swap_types_and_test() - - @use_profile("redshift") - def test__redshift__switch_materialization(self): - self.swap_types_and_test() - - @mark.flaky(rerun_filter=bigquery_rate_limiter, max_runs=3) - @use_profile("bigquery") - def test__bigquery__switch_materialization(self): - # BQ has a weird check that prevents the dropping of tables in the view materialization - # if --full-refresh is not provided. This is to prevent the clobbering of a date-sharded - # table with a view if a model config is accidently changed. We should probably remove that check - # and then remove these bq-specific tests - - results = self.run_dbt(['run', '--vars', 'materialized: view']) - self.assertEqual(results[0].node.config.materialized, 'view') - self.assertEqual(len(results), 1) - - results = self.run_dbt(['run', '--vars', 'materialized: table']) - self.assertEqual(results[0].node.config.materialized, 'table') - self.assertEqual(len(results), 1) - - results = self.run_dbt(['run', '--vars', 'materialized: view', "--full-refresh"]) - self.assertEqual(results[0].node.config.materialized, 'view') - self.assertEqual(len(results), 1) - - results = self.run_dbt(['run', '--vars', 'materialized: incremental']) - self.assertEqual(results[0].node.config.materialized, 'incremental') - self.assertEqual(len(results), 1) - - results = self.run_dbt(['run', '--vars', 'materialized: view', "--full-refresh"]) - self.assertEqual(results[0].node.config.materialized, 'view') - self.assertEqual(len(results), 1) - - @use_profile('presto') - def test__presto__switch_materialization(self): - # presto can't do incremental materializations so there's less to this - - results = self.run_dbt(['run', '--vars', 'materialized: view']) - self.assertEqual(results[0].node.config.materialized, 'view') - self.assertEqual(len(results), 1) - - results = self.run_dbt(['run', '--vars', 'materialized: table']) - self.assertEqual(results[0].node.config.materialized, 'table') - self.assertEqual(len(results), 1) - - results = self.run_dbt(['run', '--vars', 'materialized: view']) - self.assertEqual(results[0].node.config.materialized, 'view') - self.assertEqual(len(results), 1) diff --git a/test/integration/036_snowflake_view_dependency_test/data/people.csv b/test/integration/036_snowflake_view_dependency_test/data/people.csv deleted file mode 100644 index 36a69d8ca27..00000000000 --- a/test/integration/036_snowflake_view_dependency_test/data/people.csv +++ /dev/null @@ -1,4 +0,0 @@ -id,name -1,Drew -2,Jake -3,Connor diff --git a/test/integration/036_snowflake_view_dependency_test/models/.gitkeep b/test/integration/036_snowflake_view_dependency_test/models/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/test/integration/036_snowflake_view_dependency_test/models/base_table.sql b/test/integration/036_snowflake_view_dependency_test/models/base_table.sql deleted file mode 100644 index 23079234f9b..00000000000 --- a/test/integration/036_snowflake_view_dependency_test/models/base_table.sql +++ /dev/null @@ -1,9 +0,0 @@ - -{{ config(materialized='table') }} - -select * - {% if var('add_table_field', False) %} - , 1 as new_field - {% endif %} - -from {{ ref('people') }} diff --git a/test/integration/036_snowflake_view_dependency_test/models/dependent_model.sql b/test/integration/036_snowflake_view_dependency_test/models/dependent_model.sql deleted file mode 100644 index 9fef9d88f12..00000000000 --- a/test/integration/036_snowflake_view_dependency_test/models/dependent_model.sql +++ /dev/null @@ -1,8 +0,0 @@ - -{% if var('dependent_type', 'view') == 'view' %} - {{ config(materialized='view') }} -{% else %} - {{ config(materialized='table') }} -{% endif %} - -select * from {{ ref('base_table') }} diff --git a/test/integration/036_snowflake_view_dependency_test/test_view_binding_dependency.py b/test/integration/036_snowflake_view_dependency_test/test_view_binding_dependency.py deleted file mode 100644 index 61ba9baafae..00000000000 --- a/test/integration/036_snowflake_view_dependency_test/test_view_binding_dependency.py +++ /dev/null @@ -1,136 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile - -class TestSnowflakeLateBindingViewDependency(DBTIntegrationTest): - - @property - def schema(self): - return "snowflake_view_dependency_test_036" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'data-paths': ['data'], - 'seeds': { - 'quote_columns': False, - }, - 'quoting': { - 'schema': False, - 'identifier': False - } - } - - """ - Snowflake views are not bound to the relations they select from. A Snowflake view - can have entirely invalid SQL if, for example, the table it selects from is dropped - and recreated with a different schema. In these scenarios, Snowflake will raise an error if: - 1) The view is queried - 2) The view is altered - - dbt's logic should avoid running these types of queries against views in situations - where they _may_ have invalid definitions. These tests assert that views are handled - correctly in various different scenarios - """ - - @use_profile('snowflake') - def test__snowflake__changed_table_schema_for_downstream_view(self): - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 2) - self.assertManyTablesEqual(["PEOPLE", "BASE_TABLE", "DEPENDENT_MODEL"]) - - # Change the schema of base_table, assert that dependent_model doesn't fail - results = self.run_dbt(["run", "--vars", "{add_table_field: true, dependent_type: view}"]) - self.assertEqual(len(results), 2) - self.assertManyTablesEqual(["BASE_TABLE", "DEPENDENT_MODEL"]) - - """ - This test is similar to the one above, except the downstream model starts as a view, and - then is changed to be a table. This checks that the table materialization does not - errantly rename a view that might have an invalid definition, which would cause an error - """ - @use_profile('snowflake') - def test__snowflake__changed_table_schema_for_downstream_view_changed_to_table(self): - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 2) - self.assertManyTablesEqual(["PEOPLE", "BASE_TABLE", "DEPENDENT_MODEL"]) - - expected_types = { - 'base_table': 'table', - 'dependent_model': 'view' - } - - # ensure that the model actually was materialized as a table - for result in results: - node_name = result.node.name - self.assertEqual(result.node.config.materialized, expected_types[node_name]) - - results = self.run_dbt(["run", "--vars", "{add_table_field: true, dependent_type: table}"]) - self.assertEqual(len(results), 2) - self.assertManyTablesEqual(["BASE_TABLE", "DEPENDENT_MODEL"]) - - expected_types = { - 'base_table': 'table', - 'dependent_model': 'table' - } - - # ensure that the model actually was materialized as a table - for result in results: - node_name = result.node.name - self.assertEqual(result.node.config.materialized, expected_types[node_name]) - - @use_profile('presto') - def test__presto__changed_table_schema_for_downstream_view(self): - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 2) - self.assertManyTablesEqual(["people", "base_table", "dependent_model"]) - - # Change the schema of base_table, assert that dependent_model doesn't fail - results = self.run_dbt(["run", "--vars", "{add_table_field: true, dependent_type: view}"]) - self.assertEqual(len(results), 2) - self.assertManyTablesEqual(["base_table", "dependent_model"]) - - @use_profile('presto') - def test__presto__changed_table_schema_for_downstream_view_changed_to_table(self): - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 2) - self.assertManyTablesEqual(["people", "base_table", "dependent_model"]) - - expected_types = { - 'base_table': 'table', - 'dependent_model': 'view' - } - - # ensure that the model actually was materialized as a table - for result in results: - node_name = result.node.name - self.assertEqual(result.node.config.materialized, expected_types[node_name]) - - results = self.run_dbt(["run", "--vars", "{add_table_field: true, dependent_type: table}"]) - self.assertEqual(len(results), 2) - self.assertManyTablesEqual(["base_table", "dependent_model"]) - - expected_types = { - 'base_table': 'table', - 'dependent_model': 'table' - } - - # ensure that the model actually was materialized as a table - for result in results: - node_name = result.node.name - self.assertEqual(result.node.config.materialized, expected_types[node_name]) diff --git a/test/integration/038_caching_test/test_caching.py b/test/integration/038_caching_test/test_caching.py index 7ac1cf42e92..16152437db8 100644 --- a/test/integration/038_caching_test/test_caching.py +++ b/test/integration/038_caching_test/test_caching.py @@ -39,10 +39,6 @@ class TestCachingLowercaseModel(TestBaseCaching): def models(self): return "models" - @use_profile('snowflake') - def test_snowflake_cache(self): - self.cache_run() - @use_profile('postgres') def test_postgres_cache(self): self.cache_run() @@ -52,10 +48,6 @@ class TestCachingUppercaseModel(TestBaseCaching): def models(self): return "shouting_models" - @use_profile('snowflake') - def test_snowflake_cache(self): - self.cache_run() - @use_profile('postgres') def test_postgres_cache(self): self.cache_run() diff --git a/test/integration/040_override_database_test/data/seed.csv b/test/integration/040_override_database_test/data/seed.csv deleted file mode 100644 index ae9125c16dd..00000000000 --- a/test/integration/040_override_database_test/data/seed.csv +++ /dev/null @@ -1,6 +0,0 @@ -id,name -1,a -2,b -3,c -4,d -5,e diff --git a/test/integration/040_override_database_test/models/subfolder/view_3.sql b/test/integration/040_override_database_test/models/subfolder/view_3.sql deleted file mode 100644 index 4b91aa0f2fa..00000000000 --- a/test/integration/040_override_database_test/models/subfolder/view_3.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('seed') }} diff --git a/test/integration/040_override_database_test/models/subfolder/view_4.sql b/test/integration/040_override_database_test/models/subfolder/view_4.sql deleted file mode 100644 index efa1268fab8..00000000000 --- a/test/integration/040_override_database_test/models/subfolder/view_4.sql +++ /dev/null @@ -1,5 +0,0 @@ -{{ - config(database=var('alternate_db')) -}} - -select * from {{ ref('seed') }} diff --git a/test/integration/040_override_database_test/models/view_1.sql b/test/integration/040_override_database_test/models/view_1.sql deleted file mode 100644 index a43f04646b8..00000000000 --- a/test/integration/040_override_database_test/models/view_1.sql +++ /dev/null @@ -1,7 +0,0 @@ -{# - We are running against a database that must be quoted. - These calls ensure that we trigger an error if we're failing to quote at parse-time -#} -{% do adapter.already_exists(this.schema, this.table) %} -{% do adapter.get_relation(this.database, this.schema, this.table) %} -select * from {{ ref('seed') }} diff --git a/test/integration/040_override_database_test/models/view_2.sql b/test/integration/040_override_database_test/models/view_2.sql deleted file mode 100644 index 9ac6bdad6a7..00000000000 --- a/test/integration/040_override_database_test/models/view_2.sql +++ /dev/null @@ -1,6 +0,0 @@ -{%- if target.type == 'bigquery' -%} - {{ config(project=var('alternate_db')) }} -{%- else -%} - {{ config(database=var('alternate_db')) }} -{%- endif -%} -select * from {{ ref('seed') }} diff --git a/test/integration/040_override_database_test/test_override_database.py b/test/integration/040_override_database_test/test_override_database.py deleted file mode 100644 index f1f28430a87..00000000000 --- a/test/integration/040_override_database_test/test_override_database.py +++ /dev/null @@ -1,234 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile - -import os - - -class BaseOverrideDatabase(DBTIntegrationTest): - setup_alternate_db = True - @property - def schema(self): - return "override_database_040" - - @property - def models(self): - return "models" - - @property - def alternative_database(self): - if self.adapter_type == 'snowflake': - return os.getenv('SNOWFLAKE_TEST_DATABASE') - else: - return super().alternative_database - - def snowflake_profile(self): - return { - 'config': { - 'send_anonymous_usage_stats': False - }, - 'test': { - 'outputs': { - 'default2': { - 'type': 'snowflake', - 'threads': 4, - 'account': os.getenv('SNOWFLAKE_TEST_ACCOUNT'), - 'user': os.getenv('SNOWFLAKE_TEST_USER'), - 'password': os.getenv('SNOWFLAKE_TEST_PASSWORD'), - 'database': os.getenv('SNOWFLAKE_TEST_QUOTED_DATABASE'), - 'schema': self.unique_schema(), - 'warehouse': os.getenv('SNOWFLAKE_TEST_WAREHOUSE'), - }, - 'noaccess': { - 'type': 'snowflake', - 'threads': 4, - 'account': os.getenv('SNOWFLAKE_TEST_ACCOUNT'), - 'user': 'noaccess', - 'password': 'password', - 'database': os.getenv('SNOWFLAKE_TEST_DATABASE'), - 'schema': self.unique_schema(), - 'warehouse': os.getenv('SNOWFLAKE_TEST_WAREHOUSE'), - } - }, - 'target': 'default2' - } - } - - @property - def project_config(self): - return { - 'config-version': 2, - 'data-paths': ['data'], - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'quoting': { - 'database': True, - }, - 'seeds': { - 'quote_columns': False, - } - } - - -class TestModelOverride(BaseOverrideDatabase): - def run_database_override(self): - if self.adapter_type == 'snowflake': - func = lambda x: x.upper() - else: - func = lambda x: x - - self.run_dbt(['seed']) - - self.assertEqual(len(self.run_dbt(['run'])), 4) - self.assertManyRelationsEqual([ - (func('seed'), self.unique_schema(), self.default_database), - (func('view_2'), self.unique_schema(), self.alternative_database), - (func('view_1'), self.unique_schema(), self.default_database), - (func('view_3'), self.unique_schema(), self.default_database), - (func('view_4'), self.unique_schema(), self.alternative_database), - ]) - - @use_profile('bigquery') - def test_bigquery_database_override(self): - self.run_database_override() - - @use_profile('snowflake') - def test_snowflake_database_override(self): - self.run_database_override() - - -class BaseTestProjectModelOverride(BaseOverrideDatabase): - # this is janky, but I really want to access self.default_database in - # project_config - @property - def default_database(self): - target = self._profile_config['test']['target'] - profile = self._profile_config['test']['outputs'][target] - for key in ['database', 'project', 'dbname']: - if key in profile: - database = profile[key] - if self.adapter_type == 'snowflake': - return database.upper() - return database - assert False, 'No profile database found!' - - def run_database_override(self): - self.run_dbt(['seed']) - self.assertEqual(len(self.run_dbt(['run'])), 4) - self.assertExpectedRelations() - - def assertExpectedRelations(self): - if self.adapter_type == 'snowflake': - func = lambda x: x.upper() - else: - func = lambda x: x - - self.assertManyRelationsEqual([ - (func('seed'), self.unique_schema(), self.default_database), - (func('view_2'), self.unique_schema(), self.alternative_database), - (func('view_1'), self.unique_schema(), self.alternative_database), - (func('view_3'), self.unique_schema(), self.default_database), - (func('view_4'), self.unique_schema(), self.alternative_database), - ]) - - -class TestProjectModelOverride(BaseTestProjectModelOverride): - @property - def project_config(self): - return { - 'config-version': 2, - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'models': { - 'database': self.alternative_database, - 'test': { - 'subfolder': { - 'database': self.default_database, - } - } - }, - 'data-paths': ['data'], - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'quoting': { - 'database': True, - }, - 'seeds': { - 'quote_columns': False, - } - } - - @use_profile('bigquery') - def test_bigquery_database_override(self): - self.run_database_override() - - @use_profile('snowflake') - def test_snowflake_database_override(self): - self.run_database_override() - - -class TestProjectModelAliasOverride(BaseTestProjectModelOverride): - @property - def project_config(self): - return { - 'config-version': 2, - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'models': { - 'project': self.alternative_database, - 'test': { - 'subfolder': { - 'project': self.default_database, - } - } - }, - 'data-paths': ['data'], - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'quoting': { - 'database': True, - }, - 'seeds': { - 'quote_columns': False, - } - } - - @use_profile('bigquery') - def test_bigquery_project_override(self): - self.run_database_override() - - -class TestProjectSeedOverride(BaseOverrideDatabase): - def run_database_override(self): - if self.adapter_type == 'snowflake': - func = lambda x: x.upper() - else: - func = lambda x: x - - self.use_default_project({ - 'config-version': 2, - 'seeds': { - 'database': self.alternative_database - }, - }) - self.run_dbt(['seed']) - - self.assertEqual(len(self.run_dbt(['run'])), 4) - self.assertManyRelationsEqual([ - (func('seed'), self.unique_schema(), self.alternative_database), - (func('view_2'), self.unique_schema(), self.alternative_database), - (func('view_1'), self.unique_schema(), self.default_database), - (func('view_3'), self.unique_schema(), self.default_database), - (func('view_4'), self.unique_schema(), self.alternative_database), - ]) - - @use_profile('bigquery') - def test_bigquery_database_override(self): - self.run_database_override() - - @use_profile('snowflake') - def test_snowflake_database_override(self): - self.run_database_override() diff --git a/test/integration/042_sources_test/test_sources.py b/test/integration/042_sources_test/test_sources.py index f8ae2e461d7..2a43f1a0785 100644 --- a/test/integration/042_sources_test/test_sources.py +++ b/test/integration/042_sources_test/test_sources.py @@ -74,8 +74,7 @@ def _set_updated_at_to(self, delta): 'blue',{id},'Jake','abc@example.com','192.168.1.1','{time}' )""" quoted_columns = ','.join( - self.adapter.quote(c) if self.adapter_type != 'bigquery' else c - for c in + self.adapter.quote(c) for c in ('favorite_color', 'id', 'first_name', 'email', 'ip_address', 'updated_at') ) @@ -357,18 +356,6 @@ def test_postgres_source_snapshot_freshness(self): self.assertEqual(results[0].status, 'pass') self._assert_freshness_results('target/pass_source.json', 'pass') - @use_profile('snowflake') - def test_snowflake_source_freshness(self): - self._run_source_freshness() - - @use_profile('redshift') - def test_redshift_source_freshness(self): - self._run_source_freshness() - - @use_profile('bigquery') - def test_bigquery_source_freshness(self): - self._run_source_freshness() - @use_profile('postgres') def test_postgres_source_freshness_selection_select(self): """Tests node selection using the --select argument.""" @@ -492,8 +479,3 @@ def project_config(self): def test_postgres_catalog(self): self.run_dbt_with_vars(['run']) self.run_dbt_with_vars(['docs', 'generate']) - - @use_profile('redshift') - def test_redshift_catalog(self): - self.run_dbt_with_vars(['run']) - self.run_dbt_with_vars(['docs', 'generate']) diff --git a/test/integration/043_custom_aliases_test/macros-configs/macros.sql b/test/integration/043_custom_aliases_test/macros-configs/macros.sql index 13df82e812d..a50044ea09f 100644 --- a/test/integration/043_custom_aliases_test/macros-configs/macros.sql +++ b/test/integration/043_custom_aliases_test/macros-configs/macros.sql @@ -15,7 +15,3 @@ {% macro default__string_literal(s) %} '{{ s }}'::text {% endmacro %} - -{% macro bigquery__string_literal(s) %} - cast('{{ s }}' as string) -{% endmacro %} diff --git a/test/integration/043_custom_aliases_test/macros/macros.sql b/test/integration/043_custom_aliases_test/macros/macros.sql index c10575064aa..a29f223b075 100644 --- a/test/integration/043_custom_aliases_test/macros/macros.sql +++ b/test/integration/043_custom_aliases_test/macros/macros.sql @@ -15,7 +15,3 @@ {% macro default__string_literal(s) %} '{{ s }}'::text {% endmacro %} - -{% macro bigquery__string_literal(s) %} - cast('{{ s }}' as string) -{% endmacro %} diff --git a/test/integration/049_dbt_debug_test/test_debug.py b/test/integration/049_dbt_debug_test/test_debug.py index 74fabfcbc7b..8a5fbd774f3 100644 --- a/test/integration/049_dbt_debug_test/test_debug.py +++ b/test/integration/049_dbt_debug_test/test_debug.py @@ -64,11 +64,6 @@ def test_postgres_ok(self): self.run_dbt(['debug']) self.assertNotIn('ERROR', self.capsys.readouterr().out) - @use_profile('bigquery') - def test_bigquery_ok(self): - self.run_dbt(['debug']) - self.assertNotIn('ERROR', self.capsys.readouterr().out) - @use_profile('postgres') def test_postgres_nopass(self): self.run_dbt(['debug', '--target', 'nopass'], expect_pass=False) diff --git a/test/integration/050_warehouse_test/models/expected_warehouse.sql b/test/integration/050_warehouse_test/models/expected_warehouse.sql deleted file mode 100644 index a012035e6b3..00000000000 --- a/test/integration/050_warehouse_test/models/expected_warehouse.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(materialized='table') }} -select '{{ env_var("SNOWFLAKE_TEST_ALT_WAREHOUSE", "DBT_TEST_ALT") }}' as warehouse diff --git a/test/integration/050_warehouse_test/models/invalid_warehouse.sql b/test/integration/050_warehouse_test/models/invalid_warehouse.sql deleted file mode 100644 index 698059ff01b..00000000000 --- a/test/integration/050_warehouse_test/models/invalid_warehouse.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(snowflake_warehouse='DBT_TEST_DOES_NOT_EXIST') }} -select current_warehouse() as warehouse diff --git a/test/integration/050_warehouse_test/models/override_warehouse.sql b/test/integration/050_warehouse_test/models/override_warehouse.sql deleted file mode 100644 index 957718d2539..00000000000 --- a/test/integration/050_warehouse_test/models/override_warehouse.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(snowflake_warehouse=env_var('SNOWFLAKE_TEST_ALT_WAREHOUSE', 'DBT_TEST_ALT'), materialized='table') }} -select current_warehouse() as warehouse diff --git a/test/integration/050_warehouse_test/project-config-models/expected_warehouse.sql b/test/integration/050_warehouse_test/project-config-models/expected_warehouse.sql deleted file mode 100644 index a012035e6b3..00000000000 --- a/test/integration/050_warehouse_test/project-config-models/expected_warehouse.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(materialized='table') }} -select '{{ env_var("SNOWFLAKE_TEST_ALT_WAREHOUSE", "DBT_TEST_ALT") }}' as warehouse diff --git a/test/integration/050_warehouse_test/project-config-models/override_warehouse.sql b/test/integration/050_warehouse_test/project-config-models/override_warehouse.sql deleted file mode 100644 index e3d38eb5159..00000000000 --- a/test/integration/050_warehouse_test/project-config-models/override_warehouse.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(materialized='table') }} -select current_warehouse() as warehouse diff --git a/test/integration/050_warehouse_test/test_warehouses.py b/test/integration/050_warehouse_test/test_warehouses.py deleted file mode 100644 index eaef4ddb8ac..00000000000 --- a/test/integration/050_warehouse_test/test_warehouses.py +++ /dev/null @@ -1,62 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile -import os - - -class TestModelWarehouse(DBTIntegrationTest): - @property - def schema(self): - return 'dbt_warehouse_050' - - @staticmethod - def dir(value): - return os.path.normpath(value) - - @property - def models(self): - return self.dir('models') - - @use_profile('snowflake') - def test_snowflake_override_ok(self): - self.run_dbt([ - 'run', - '--models', 'override_warehouse', 'expected_warehouse', - ]) - self.assertManyRelationsEqual([['OVERRIDE_WAREHOUSE'], ['EXPECTED_WAREHOUSE']]) - - @use_profile('snowflake') - def test_snowflake_override_noexist(self): - self.run_dbt(['run', '--models', 'invalid_warehouse'], expect_pass=False) - - -class TestConfigWarehouse(DBTIntegrationTest): - @property - def schema(self): - return 'dbt_warehouse_050' - - @property - def project_config(self): - return { - 'config-version': 2, - 'source-paths': ['project-config-models'], - 'models': { - 'test': { - 'snowflake_warehouse': os.getenv('SNOWFLAKE_TEST_ALT_WAREHOUSE', 'DBT_TEST_ALT'), - }, - }, - } - - @staticmethod - def dir(value): - return os.path.normpath(value) - - @property - def models(self): - return self.dir('models') - - @use_profile('snowflake') - def test_snowflake_override_ok(self): - self.run_dbt([ - 'run', - '--models', 'override_warehouse', 'expected_warehouse', - ]) - self.assertManyRelationsEqual([['OVERRIDE_WAREHOUSE'], ['EXPECTED_WAREHOUSE']]) diff --git a/test/integration/051_query_comments_test/models/x.sql b/test/integration/051_query_comments_test/models/x.sql index d103fc6f92c..00b76f02d3c 100644 --- a/test/integration/051_query_comments_test/models/x.sql +++ b/test/integration/051_query_comments_test/models/x.sql @@ -12,15 +12,10 @@ {% set required = ['name', 'schema', 'type', 'threads'] %} {# Require what we docuement at https://docs.getdbt.com/docs/target #} -{% if target.type == 'postgres' or target.type == 'redshift' %} +{% if target.type == 'postgres' %} {% do required.extend(['dbname', 'host', 'user', 'port']) %} -{% elif target.type == 'snowflake' %} - {% do required.extend(['database', 'warehouse', 'user', 'role', 'account']) %} -{% elif target.type == 'bigquery' %} - {% do required.extend(['project']) %} {% else %} {% do exceptions.raise_compiler_error('invalid target, got unknown type "' ~ target.type ~ '"') %} - {% endif %} {% for value in required %} diff --git a/test/integration/051_query_comments_test/test_query_comments.py b/test/integration/051_query_comments_test/test_query_comments.py index 54c211f2f63..6eed09f416a 100644 --- a/test/integration/051_query_comments_test/test_query_comments.py +++ b/test/integration/051_query_comments_test/test_query_comments.py @@ -94,18 +94,6 @@ def run_assert_comments(self): def test_postgres_comments(self): self.run_assert_comments() - @use_profile('redshift') - def test_redshift_comments(self): - self.run_assert_comments() - - @use_profile('snowflake') - def test_snowflake_comments(self): - self.run_assert_comments() - - @use_profile('bigquery') - def test_bigquery_comments(self): - self.run_assert_comments() - class TestQueryComments(TestDefaultQueryComments): @property diff --git a/test/integration/052_column_quoting/models-unquoted/model.sql b/test/integration/052_column_quoting/models-unquoted/model.sql index e6862aa2e17..1bdcda38353 100644 --- a/test/integration/052_column_quoting/models-unquoted/model.sql +++ b/test/integration/052_column_quoting/models-unquoted/model.sql @@ -1,12 +1,5 @@ {% set col_a = '"col_a"' %} {% set col_b = '"col_b"' %} -{% if adapter.type() == 'bigquery' %} - {% set col_a = '`col_a`' %} - {% set col_b = '`col_b`' %} -{% elif adapter.type() == 'snowflake' %} - {% set col_a = '"COL_A"' %} - {% set col_b = '"COL_B"' %} -{% endif %} {{config( materialized = 'incremental', diff --git a/test/integration/052_column_quoting/models/model.sql b/test/integration/052_column_quoting/models/model.sql index 8c19c6546a2..3bc61e082d9 100644 --- a/test/integration/052_column_quoting/models/model.sql +++ b/test/integration/052_column_quoting/models/model.sql @@ -1,9 +1,5 @@ {% set col_a = '"col_A"' %} {% set col_b = '"col_B"' %} -{% if adapter.type() == 'bigquery' %} - {% set col_a = '`col_A`' %} - {% set col_b = '`col_B`' %} -{% endif %} {{config( materialized = 'incremental', diff --git a/test/integration/052_column_quoting/test_column_quotes.py b/test/integration/052_column_quoting/test_column_quotes.py index 7d7d3ade96b..f5aef6fed39 100644 --- a/test/integration/052_column_quoting/test_column_quotes.py +++ b/test/integration/052_column_quoting/test_column_quotes.py @@ -39,33 +39,6 @@ def run_dbt(self, *args, **kwargs): def test_postgres_column_quotes(self): self._run_columnn_quotes() - @use_profile('redshift') - def test_redshift_column_quotes(self): - self._run_columnn_quotes() - - @use_profile('bigquery') - def test_bigquery_column_quotes(self): - self._run_columnn_quotes(strategy='merge') - - -class TestColumnQuotingSnowflakeDefault(BaseColumnQuotingTest): - @property - def project_config(self): - return { - 'config-version': 2 - } - - @property - def models(self): - return self.dir('models-unquoted') - - def run_dbt(self, *args, **kwargs): - return super().run_dbt(*args, **kwargs) - - @use_profile('snowflake') - def test_snowflake_column_quotes(self): - self._run_columnn_quotes() - class TestColumnQuotingDisabled(BaseColumnQuotingTest): @property @@ -85,22 +58,6 @@ def project_config(self): def test_postgres_column_quotes(self): self._run_columnn_quotes() - @use_profile('redshift') - def test_redshift_column_quotes(self): - self._run_columnn_quotes() - - @use_profile('snowflake') - def test_snowflake_column_quotes(self): - self._run_columnn_quotes() - - @use_profile('snowflake') - def test_snowflake_column_quotes_merged(self): - self._run_columnn_quotes(strategy='merge') - - @use_profile('bigquery') - def test_bigquery_column_quotes_merged(self): - self._run_columnn_quotes(strategy='merge') - class TestColumnQuotingEnabled(BaseColumnQuotingTest): @property @@ -119,19 +76,3 @@ def project_config(self): @use_profile('postgres') def test_postgres_column_quotes(self): self._run_columnn_quotes() - - @use_profile('redshift') - def test_redshift_column_quotes(self): - self._run_columnn_quotes() - - @use_profile('snowflake') - def test_snowflake_column_quotes(self): - self._run_columnn_quotes() - - @use_profile('snowflake') - def test_snowflake_column_quotes_merged(self): - self._run_columnn_quotes(strategy='merge') - - @use_profile('bigquery') - def test_bigquery_column_quotes_merged(self): - self._run_columnn_quotes(strategy='merge') diff --git a/test/integration/054_adapter_methods_test/bigquery-models/renamed_model.sql b/test/integration/054_adapter_methods_test/bigquery-models/renamed_model.sql deleted file mode 100644 index 3c030d920ec..00000000000 --- a/test/integration/054_adapter_methods_test/bigquery-models/renamed_model.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ source('test_source', 'renamed_seed') }} \ No newline at end of file diff --git a/test/integration/054_adapter_methods_test/bigquery-models/sources.yml b/test/integration/054_adapter_methods_test/bigquery-models/sources.yml deleted file mode 100644 index cb74cde9c05..00000000000 --- a/test/integration/054_adapter_methods_test/bigquery-models/sources.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: 2 -sources: - - name: test_source - schema: "{{ target.schema }}" - tables: - - name: renamed_seed diff --git a/test/integration/054_adapter_methods_test/macros/rename_named_relation.sql b/test/integration/054_adapter_methods_test/macros/rename_named_relation.sql deleted file mode 100644 index 253e1e0ad8a..00000000000 --- a/test/integration/054_adapter_methods_test/macros/rename_named_relation.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Macro to rename a relation -{% macro rename_named_relation(from_name, to_name) %} -{%- set from_relation = api.Relation.create(database=target.database, schema=target.schema, identifier=from_name, type='table') -%} -{%- set to_relation = api.Relation.create(database=target.database, schema=target.schema, identifier=to_name, type='table') -%} -{% do adapter.rename_relation(from_relation, to_relation) %} -{% endmacro %} \ No newline at end of file diff --git a/test/integration/054_adapter_methods_test/seed_bq.sql b/test/integration/054_adapter_methods_test/seed_bq.sql deleted file mode 100644 index 71a9a78c665..00000000000 --- a/test/integration/054_adapter_methods_test/seed_bq.sql +++ /dev/null @@ -1,32 +0,0 @@ -create table {database}.{schema}.seed ( - id INT64, - first_name STRING, - last_name STRING, - email STRING, - gender STRING, - ip_address STRING, - updated_at TIMESTAMP -); - --- seed inserts -insert {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values -(1, 'Judith', 'Kennedy', 'jkennedy0@phpbb.com', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), -(2, 'Arthur', 'Kelly', 'akelly1@eepurl.com', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), -(3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'), -(4, 'Ralph', 'Turner', 'rturner3@hp.com', 'Male', '157.83.76.114', '2016-08-08 00:06:51'), -(5, 'Laura', 'Gonzales', 'lgonzales4@howstuffworks.com', 'Female', '30.54.105.168', '2016-09-01 08:25:38'), -(6, 'Katherine', 'Lopez', 'klopez5@yahoo.co.jp', 'Female', '169.138.46.89', '2016-08-30 18:52:11'), -(7, 'Jeremy', 'Hamilton', 'jhamilton6@mozilla.org', 'Male', '231.189.13.133', '2016-07-17 02:09:46'), -(8, 'Heather', 'Rose', 'hrose7@goodreads.com', 'Female', '87.165.201.65', '2015-12-29 22:03:56'), -(9, 'Gregory', 'Kelly', 'gkelly8@trellian.com', 'Male', '154.209.99.7', '2016-03-24 21:18:16'), -(10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '2016-08-20 15:44:49'), -(11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '2016-02-27 01:41:48'), -(12, 'Russell', 'Lawrence', 'rlawrenceb@qq.com', 'Male', '189.115.73.4', '2016-06-11 03:07:09'), -(13, 'Michelle', 'Montgomery', 'mmontgomeryc@scientificamerican.com', 'Female', '243.220.95.82', '2016-06-18 16:27:19'), -(14, 'Walter', 'Castillo', 'wcastillod@pagesperso-orange.fr', 'Male', '71.159.238.196', '2016-10-06 01:55:44'), -(15, 'Robin', 'Mills', 'rmillse@vkontakte.ru', 'Female', '172.190.5.50', '2016-10-31 11:41:21'), -(16, 'Raymond', 'Holmes', 'rholmesf@usgs.gov', 'Male', '148.153.166.95', '2016-10-03 08:16:38'), -(17, 'Gary', 'Bishop', 'gbishopg@plala.or.jp', 'Male', '161.108.182.13', '2016-08-29 19:35:20'), -(18, 'Anna', 'Riley', 'arileyh@nasa.gov', 'Female', '253.31.108.22', '2015-12-11 04:34:27'), -(19, 'Sarah', 'Knight', 'sknighti@foxnews.com', 'Female', '222.220.3.177', '2016-09-26 00:49:06'), -(20, 'Phyllis', 'Fox', null, 'Female', '163.191.232.95', '2016-08-21 10:35:19'); diff --git a/test/integration/054_adapter_methods_test/test_adapter_methods.py b/test/integration/054_adapter_methods_test/test_adapter_methods.py index 1fcaf5b1a49..9093a7b76c3 100644 --- a/test/integration/054_adapter_methods_test/test_adapter_methods.py +++ b/test/integration/054_adapter_methods_test/test_adapter_methods.py @@ -5,7 +5,7 @@ class TestBaseCaching(DBTIntegrationTest): @property def schema(self): - return "caching_038" + return "adapter_methods_caching" @property def models(self): @@ -23,99 +23,3 @@ def test_postgres_adapter_methods(self): self.run_dbt(['compile']) # trigger any compile-time issues self.run_dbt() self.assertTablesEqual('model', 'expected') - - @use_profile('redshift') - def test_redshift_adapter_methods(self): - self.run_dbt(['compile']) # trigger any compile-time issues - self.run_dbt() - self.assertTablesEqual('model', 'expected') - - @use_profile('snowflake') - def test_snowflake_adapter_methods(self): - self.run_dbt(['compile']) # trigger any compile-time issues - self.run_dbt() - self.assertTablesEqual('MODEL', 'EXPECTED') - - @use_profile('bigquery') - def test_bigquery_adapter_methods(self): - self.run_dbt(['compile']) # trigger any compile-time issues - self.run_dbt() - self.assertTablesEqual('model', 'expected') - - -class TestRenameRelation(DBTIntegrationTest): - @property - def schema(self): - return "rename_relation_054" - - @property - def models(self): - return 'bigquery-models' - - @property - def project_config(self): - return { - 'config-version': 2, - 'source-paths': ['models'] - } - - @use_profile('bigquery') - def test_bigquery_adapter_methods(self): - self.run_dbt(['compile']) # trigger any compile-time issues - self.run_sql_file("seed_bq.sql") - self.run_dbt(['seed']) - rename_relation_args = yaml.safe_dump({ - 'from_name': 'seed', - 'to_name': 'renamed_seed', - }) - self.run_dbt(['run-operation', 'rename_named_relation', '--args', rename_relation_args]) - self.run_dbt() - - -class TestGrantAccess(DBTIntegrationTest): - @property - def schema(self): - return "grant_access_054" - - @property - def models(self): - return 'bigquery-models' - - @property - def project_config(self): - return { - 'config-version': 2, - 'source-paths': ['models'] - } - - @use_profile('bigquery') - def test_bigquery_adapter_methods(self): - from dbt.adapters.bigquery import GrantTarget - from google.cloud.bigquery import AccessEntry - - self.run_dbt(['compile']) # trigger any compile-time issues - self.run_sql_file("seed_bq.sql") - self.run_dbt(['seed']) - - ae_role = "READER" - ae_entity = "user@email.com" - ae_entity_type = "userByEmail" - ae_grant_target_dict = { - 'project': self.default_database, - 'dataset': self.unique_schema() - } - self.adapter.grant_access_to(ae_entity, ae_entity_type, ae_role, ae_grant_target_dict) - - conn = self.adapter.connections.get_thread_connection() - client = conn.handle - - grant_target = GrantTarget.from_dict(ae_grant_target_dict) - dataset = client.get_dataset( - self.adapter.connections.dataset_from_id(grant_target.render()) - ) - - expected_access_entry = AccessEntry(ae_role, ae_entity_type, ae_entity) - self.assertTrue(expected_access_entry in dataset.access_entries) - - unexpected_access_entry = AccessEntry(ae_role, ae_entity_type, "unexpected@email.com") - self.assertFalse(unexpected_access_entry in dataset.access_entries) diff --git a/test/integration/056_column_type_tests/bq_models/model.sql b/test/integration/056_column_type_tests/bq_models/model.sql deleted file mode 100644 index 94e4fba18ca..00000000000 --- a/test/integration/056_column_type_tests/bq_models/model.sql +++ /dev/null @@ -1,5 +0,0 @@ -select - CAST(1 as int64) as int64_col, - CAST(2.0 as float64) as float64_col, - CAST(3.0 as numeric) as numeric_col, - CAST('3' as string) as string_col, diff --git a/test/integration/056_column_type_tests/bq_models/schema.yml b/test/integration/056_column_type_tests/bq_models/schema.yml deleted file mode 100644 index 8eb8a9ae286..00000000000 --- a/test/integration/056_column_type_tests/bq_models/schema.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: 2 -models: - - name: model - tests: - - is_type: - column_map: - int64_col: ['integer', 'number'] - float64_col: ['float', 'number'] - numeric_col: ['numeric', 'number'] - string_col: ['string', 'not number'] diff --git a/test/integration/056_column_type_tests/bq_models_alter_type/altered_schema.yml b/test/integration/056_column_type_tests/bq_models_alter_type/altered_schema.yml deleted file mode 100644 index 3a78da404b2..00000000000 --- a/test/integration/056_column_type_tests/bq_models_alter_type/altered_schema.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: 2 -models: - - name: model - tests: - - is_type: - column_map: - int64_col: ['string', 'not number'] - float64_col: ['float', 'number'] - numeric_col: ['numeric', 'number'] - string_col: ['string', 'not number'] diff --git a/test/integration/056_column_type_tests/bq_models_alter_type/model.sql b/test/integration/056_column_type_tests/bq_models_alter_type/model.sql deleted file mode 100644 index 066252ae5aa..00000000000 --- a/test/integration/056_column_type_tests/bq_models_alter_type/model.sql +++ /dev/null @@ -1,6 +0,0 @@ -{{ config(materialized='table') }} -select - CAST(1 as int64) as int64_col, - CAST(2.0 as float64) as float64_col, - CAST(3.0 as numeric) as numeric_col, - CAST('3' as string) as string_col, diff --git a/test/integration/056_column_type_tests/rs_models/model.sql b/test/integration/056_column_type_tests/rs_models/model.sql deleted file mode 100644 index f8e9721030a..00000000000 --- a/test/integration/056_column_type_tests/rs_models/model.sql +++ /dev/null @@ -1,17 +0,0 @@ -select - 1::smallint as smallint_col, - 2::int as int_col, - 3::bigint as bigint_col, - 4::int2 as int2_col, - 5::int4 as int4_col, - 6::int8 as int8_col, - 7::integer as integer_col, - 8.0::real as real_col, - 9.0::float4 as float4_col, - 10.0::float8 as float8_col, - 11.0::float as float_col, - 12.0::double precision as double_col, - 13.0::numeric as numeric_col, - 14.0::decimal as decimal_col, - '15'::varchar(20) as varchar_col, - '16'::text as text_col diff --git a/test/integration/056_column_type_tests/rs_models/schema.yml b/test/integration/056_column_type_tests/rs_models/schema.yml deleted file mode 100644 index 5b35ce02526..00000000000 --- a/test/integration/056_column_type_tests/rs_models/schema.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 2 -models: - - name: model - tests: - - is_type: - column_map: - smallint_col: ['integer', 'number'] - int_col: ['integer', 'number'] - bigint_col: ['integer', 'number'] - int2_col: ['integer', 'number'] - int4_col: ['integer', 'number'] - int8_col: ['integer', 'number'] - integer_col: ['integer', 'number'] - real_col: ['float', 'number'] - double_col: ['float', 'number'] - float4_col: ['float', 'number'] - float8_col: ['float', 'number'] - float_col: ['float', 'number'] - numeric_col: ['numeric', 'number'] - decimal_col: ['numeric', 'number'] - varchar_col: ['string', 'not number'] - text_col: ['string', 'not number'] diff --git a/test/integration/056_column_type_tests/sf_models/model.sql b/test/integration/056_column_type_tests/sf_models/model.sql deleted file mode 100644 index b58c7c43cb0..00000000000 --- a/test/integration/056_column_type_tests/sf_models/model.sql +++ /dev/null @@ -1,18 +0,0 @@ -select - 1::smallint as smallint_col, - 2::int as int_col, - 3::bigint as bigint_col, - 4::integer as integer_col, - 5::tinyint as tinyint_col, - 6::byteint as byteint_col, - 7.0::float as float_col, - 8.0::float4 as float4_col, - 9.0::float8 as float8_col, - 10.0::double as double_col, - 11.0::double precision as double_p_col, - 12.0::real as real_col, - 13.0::numeric as numeric_col, - 14.0::decimal as decimal_col, - 15.0::number as number_col, - '16'::text as text_col, - '17'::varchar(20) as varchar_col diff --git a/test/integration/056_column_type_tests/sf_models/schema.yml b/test/integration/056_column_type_tests/sf_models/schema.yml deleted file mode 100644 index 5be21188c5e..00000000000 --- a/test/integration/056_column_type_tests/sf_models/schema.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: 2 -models: - - name: model - tests: - - is_type: - column_map: - smallint_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] - int_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] - bigint_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] - integer_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] - tinyint_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] - byteint_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] - float_col: ['float', 'number', 'not string', 'not integer', 'not numeric'] - float4_col: ['float', 'number', 'not string', 'not integer', 'not numeric'] - float8_col: ['float', 'number', 'not string', 'not integer', 'not numeric'] - double_col: ['float', 'number', 'not string', 'not integer', 'not numeric'] - double_p_col: ['float', 'number', 'not string', 'not integer', 'not numeric'] - real_col: ['float', 'number', 'not string', 'not integer', 'not numeric'] - numeric_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] - decimal_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] - number_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] - text_col: ['string', 'not number'] - varchar_col: ['string', 'not number'] diff --git a/test/integration/056_column_type_tests/test_alter_column_types.py b/test/integration/056_column_type_tests/test_alter_column_types.py index 51970244b84..e06e1f5697c 100644 --- a/test/integration/056_column_type_tests/test_alter_column_types.py +++ b/test/integration/056_column_type_tests/test_alter_column_types.py @@ -11,18 +11,3 @@ def run_and_alter_and_test(self, alter_column_type_args): self.assertEqual(len(self.run_dbt(['run'])), 1) self.run_dbt(['run-operation', 'test_alter_column_type', '--args', alter_column_type_args]) self.assertEqual(len(self.run_dbt(['test'])), 1) - - -class TestBigQueryAlterColumnTypes(TestAlterColumnTypes): - @property - def models(self): - return 'bq_models_alter_type' - - @use_profile('bigquery') - def test_bigquery_column_types(self): - alter_column_type_args = yaml.safe_dump({ - 'model_name': 'model', - 'column_name': 'int64_col', - 'new_column_type': 'string' - }) - self.run_and_alter_and_test(alter_column_type_args) diff --git a/test/integration/056_column_type_tests/test_column_types.py b/test/integration/056_column_type_tests/test_column_types.py index 90c611169bb..66abbb4c970 100644 --- a/test/integration/056_column_type_tests/test_column_types.py +++ b/test/integration/056_column_type_tests/test_column_types.py @@ -1,4 +1,3 @@ -import pytest from test.integration.base import DBTIntegrationTest, use_profile @@ -21,31 +20,3 @@ def models(self): def test_postgres_column_types(self): self.run_and_test() -class TestRedshiftColumnTypes(TestColumnTypes): - @property - def models(self): - return 'rs_models' - - @use_profile('redshift') - def test_redshift_column_types(self): - self.run_and_test() - - -class TestSnowflakeColumnTypes(TestColumnTypes): - @property - def models(self): - return 'sf_models' - - @use_profile('snowflake') - def test_snowflake_column_types(self): - self.run_and_test() - - -class TestBigQueryColumnTypes(TestColumnTypes): - @property - def models(self): - return 'bq_models' - - @use_profile('bigquery') - def test_bigquery_column_types(self): - self.run_and_test() diff --git a/test/integration/057_oauth_tests/models/model_1.sql b/test/integration/057_oauth_tests/models/model_1.sql deleted file mode 100644 index 43258a71464..00000000000 --- a/test/integration/057_oauth_tests/models/model_1.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as id diff --git a/test/integration/057_oauth_tests/models/model_2.sql b/test/integration/057_oauth_tests/models/model_2.sql deleted file mode 100644 index 33560d6c082..00000000000 --- a/test/integration/057_oauth_tests/models/model_2.sql +++ /dev/null @@ -1 +0,0 @@ -select 2 as id diff --git a/test/integration/057_oauth_tests/models/model_3.sql b/test/integration/057_oauth_tests/models/model_3.sql deleted file mode 100644 index c724eec6fa9..00000000000 --- a/test/integration/057_oauth_tests/models/model_3.sql +++ /dev/null @@ -1,3 +0,0 @@ -select * from {{ ref('model_1') }} -union all -select * from {{ ref('model_2') }} diff --git a/test/integration/057_oauth_tests/models/model_4.sql b/test/integration/057_oauth_tests/models/model_4.sql deleted file mode 100644 index 2e8896da085..00000000000 --- a/test/integration/057_oauth_tests/models/model_4.sql +++ /dev/null @@ -1,3 +0,0 @@ -select 1 as id -union all -select 2 as id diff --git a/test/integration/057_oauth_tests/test_oauth.py b/test/integration/057_oauth_tests/test_oauth.py deleted file mode 100644 index bc8bd344f2e..00000000000 --- a/test/integration/057_oauth_tests/test_oauth.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -The first time using an account for testing, you should run this: - -``` -CREATE OR REPLACE SECURITY INTEGRATION DBT_INTEGRATION_TEST_OAUTH - TYPE = OAUTH - ENABLED = TRUE - OAUTH_CLIENT = CUSTOM - OAUTH_CLIENT_TYPE = 'CONFIDENTIAL' - OAUTH_REDIRECT_URI = 'http://localhost:8080' - oauth_issue_refresh_tokens = true - OAUTH_ALLOW_NON_TLS_REDIRECT_URI = true - BLOCKED_ROLES_LIST = - oauth_refresh_token_validity = 7776000; -``` - - -Every month (or any amount <90 days): - -Run `select SYSTEM$SHOW_OAUTH_CLIENT_SECRETS('DBT_INTEGRATION_TEST_OAUTH');` - -The only row/column of output should be a json blob, it goes (within single -quotes!) as the second argument to the server script: - -python scripts/werkzeug-refresh-token.py ${acount_name} '${json_blob}' - -Open http://localhost:8080 - -Log in as the test user, get a resonse page with some environment variables. -Update CI providers and test.env with the new values (If you kept the security -integration the same, just the refresh token changed) -""" -import os -import pytest -from test.integration.base import DBTIntegrationTest, use_profile - - -def env_set_truthy(key): - """Return the value if it was set to a "truthy" string value, or None - otherwise. - """ - value = os.getenv(key) - if not value or value.lower() in ('0', 'false', 'f'): - return None - return value - - -OAUTH_TESTS_DISABLED = env_set_truthy('DBT_INTEGRATION_TEST_SNOWFLAKE_OAUTH_DISABLED') - - -class TestSnowflakeOauth(DBTIntegrationTest): - @property - def schema(self): - return "simple_copy_001" - - @staticmethod - def dir(path): - return path.lstrip('/') - - @property - def models(self): - return self.dir("models") - - def snowflake_profile(self): - profile = super().snowflake_profile() - profile['test']['target'] = 'oauth' - missing = ', '.join( - k for k in ('oauth_client_id', 'oauth_client_secret', 'token') - if k not in profile['test']['outputs']['oauth'] - ) - if missing: - raise ValueError(f'Cannot run test - {missing} not configured') - del profile['test']['outputs']['default2'] - del profile['test']['outputs']['noaccess'] - return profile - - @pytest.mark.skipif(OAUTH_TESTS_DISABLED, reason='oauth tests disabled') - @use_profile('snowflake') - def test_snowflake_basic(self): - self.run_dbt() - self.assertManyRelationsEqual([['MODEL_3'], ['MODEL_4']]) diff --git a/test/integration/059_source_overrides_test/test_source_overrides.py b/test/integration/059_source_overrides_test/test_source_overrides.py index cd55efd595c..0e4628acc31 100644 --- a/test/integration/059_source_overrides_test/test_source_overrides.py +++ b/test/integration/059_source_overrides_test/test_source_overrides.py @@ -59,8 +59,7 @@ def _set_updated_at_to(self, delta): 'blue',{id},'Jake','abc@example.com','192.168.1.1','{time}' )""" quoted_columns = ','.join( - self.adapter.quote(c) if self.adapter_type != 'bigquery' else c - for c in + self.adapter.quote(c) for c in ('favorite_color', 'id', 'first_name', 'email', 'ip_address', 'updated_at') ) self.run_sql( diff --git a/test/integration/060_persist_docs_tests/data/seed.csv b/test/integration/060_persist_docs_tests/data/seed.csv deleted file mode 100644 index 1a728c8ab74..00000000000 --- a/test/integration/060_persist_docs_tests/data/seed.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,name -1,Alice -2,Bob diff --git a/test/integration/060_persist_docs_tests/models-bigquery-nested/schema.yml b/test/integration/060_persist_docs_tests/models-bigquery-nested/schema.yml deleted file mode 100644 index 0311dcb1449..00000000000 --- a/test/integration/060_persist_docs_tests/models-bigquery-nested/schema.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: 2 - -models: - - name: table_model_nested - columns: - - name: level_1 - description: level_1 column description - - name: level_1.level_2 - description: level_2 column description - - name: level_1.level_2.level_3_a - description: level_3 column description - - name: view_model_nested - columns: - - name: level_1 - description: level_1 column description - - name: level_1.level_2 - description: level_2 column description - - name: level_1.level_2.level_3_a - description: level_3 column description \ No newline at end of file diff --git a/test/integration/060_persist_docs_tests/models-bigquery-nested/table_model_nested.sql b/test/integration/060_persist_docs_tests/models-bigquery-nested/table_model_nested.sql deleted file mode 100644 index c2936d4f186..00000000000 --- a/test/integration/060_persist_docs_tests/models-bigquery-nested/table_model_nested.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ config(materialized='table') }} -SELECT - STRUCT( - STRUCT( - 1 AS level_3_a, - 2 AS level_3_b - ) AS level_2 - ) AS level_1 \ No newline at end of file diff --git a/test/integration/060_persist_docs_tests/models-bigquery-nested/view_model_nested.sql b/test/integration/060_persist_docs_tests/models-bigquery-nested/view_model_nested.sql deleted file mode 100644 index e3323ddf4e6..00000000000 --- a/test/integration/060_persist_docs_tests/models-bigquery-nested/view_model_nested.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ config(materialized='view') }} -SELECT - STRUCT( - STRUCT( - 1 AS level_3_a, - 2 AS level_3_b - ) AS level_2 - ) AS level_1 \ No newline at end of file diff --git a/test/integration/060_persist_docs_tests/models-column-missing/missing_column.sql b/test/integration/060_persist_docs_tests/models-column-missing/missing_column.sql deleted file mode 100644 index 642b0f14a19..00000000000 --- a/test/integration/060_persist_docs_tests/models-column-missing/missing_column.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(materialized='table') }} -select 1 as id, 'Ed' as name diff --git a/test/integration/060_persist_docs_tests/models-column-missing/schema.yml b/test/integration/060_persist_docs_tests/models-column-missing/schema.yml deleted file mode 100644 index aa7b4f88820..00000000000 --- a/test/integration/060_persist_docs_tests/models-column-missing/schema.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: 2 -models: - - name: missing_column - columns: - - name: id - description: "test id column description" - - name: column_that_does_not_exist - description: "comment that cannot be created" diff --git a/test/integration/060_persist_docs_tests/test_persist_docs.py b/test/integration/060_persist_docs_tests/test_persist_docs.py index 116818c0df9..a2e298294f0 100644 --- a/test/integration/060_persist_docs_tests/test_persist_docs.py +++ b/test/integration/060_persist_docs_tests/test_persist_docs.py @@ -95,227 +95,3 @@ def run_has_comments_pglike(self): def test_postgres_comments(self): self.run_has_comments_pglike() - @use_profile('redshift') - def test_redshift_comments(self): - self.run_has_comments_pglike() - - @use_profile('snowflake') - def test_snowflake_comments(self): - self.run_dbt() - self.run_dbt(['docs', 'generate']) - with open('target/catalog.json') as fp: - catalog_data = json.load(fp) - assert 'nodes' in catalog_data - assert len(catalog_data['nodes']) == 3 - table_node = catalog_data['nodes']['model.test.table_model'] - table_comment = table_node['metadata']['comment'] - assert table_comment.startswith('Table model description') - - table_id_comment = table_node['columns']['ID']['comment'] - assert table_id_comment.startswith('id Column description') - - table_name_comment = table_node['columns']['NAME']['comment'] - assert table_name_comment.startswith( - 'Some stuff here and then a call to') - - - -class TestPersistDocsLateBinding(BasePersistDocsTest): - @property - def project_config(self): - return { - 'config-version': 2, - 'models': { - 'test': { - '+persist_docs': { - "relation": True, - "columns": True, - }, - 'view_model': { - 'bind': False, - } - } - } - } - - @use_profile('redshift') - def test_redshift_late_binding_view(self): - self.run_dbt() - self.run_dbt(['docs', 'generate']) - with open('target/catalog.json') as fp: - catalog_data = json.load(fp) - assert 'nodes' in catalog_data - assert len(catalog_data['nodes']) == 3 - table_node = catalog_data['nodes']['model.test.table_model'] - view_node = self._assert_has_table_comments(table_node) - - view_node = catalog_data['nodes']['model.test.view_model'] - self._assert_has_view_comments(view_node, False, False) - - no_docs_node = catalog_data['nodes']['model.test.no_docs_model'] - self._assert_has_view_comments(no_docs_node, False, False) - - -class TestPersistDocsSimple(BasePersistDocsTest): - @property - def project_config(self): - return { - 'config-version': 2, - 'models': { - 'test': { - '+persist_docs': { - "relation": True, - "columns": True, - }, - } - }, - 'seeds': { - 'test': { - '+persist_docs': { - "relation": True, - "columns": True, - }, - } - }, - } - - @use_profile('snowflake') - def test_snowflake_persist_docs(self): - self.run_dbt() - - @use_profile('bigquery') - def test_bigquery_persist_docs(self): - self.run_dbt(['seed']) - self.run_dbt() - desc_map = { - 'seed': 'Seed model description', - 'table_model': 'Table model description', - 'view_model': 'View model description', - } - for node_id in ['seed', 'table_model', 'view_model']: - with self.adapter.connection_named('_test'): - client = self.adapter.connections \ - .get_thread_connection().handle - - table_id = "{}.{}.{}".format( - self.default_database, - self.unique_schema(), - node_id - ) - bq_table = client.get_table(table_id) - - bq_schema = bq_table.schema - - assert bq_table.description.startswith(desc_map[node_id]) - assert bq_schema[0].description.startswith('id Column description ') - if not node_id.startswith('view'): - assert bq_schema[1].description.startswith('Some stuff here and then a call to') - - -class TestPersistDocsNested(BasePersistDocsTest): - @property - def project_config(self): - return { - 'config-version': 2, - 'models': { - 'test': { - '+persist_docs': { - "relation": True, - "columns": True, - }, - } - } - } - - @property - def models(self): - return 'models-bigquery-nested' - - @use_profile('bigquery') - def test_bigquery_persist_docs(self): - """ - run dbt and use the bigquery client from the adapter to check if the - colunmn descriptions are persisted on the test model table and view. - - Next, generate the catalog and check if the comments are also included. - """ - self.run_dbt(['seed']) - self.run_dbt() - - self.run_dbt(['docs', 'generate']) - with open('target/catalog.json') as fp: - catalog_data = json.load(fp) - assert 'nodes' in catalog_data - assert len(catalog_data['nodes']) == 3 # seed, table, and view model - - for node_id in ['table_model_nested', 'view_model_nested']: - # check the descriptions using the api - with self.adapter.connection_named('_test'): - client = self.adapter.connections \ - .get_thread_connection().handle - - table_id = "{}.{}.{}".format( - self.default_database, - self.unique_schema(), - node_id - ) - bq_schema = client.get_table(table_id).schema - - level_1_field = bq_schema[0] - assert level_1_field.description == \ - "level_1 column description" - - level_2_field = level_1_field.fields[0] - assert level_2_field.description == \ - "level_2 column description" - - level_3_field = level_2_field.fields[0] - assert level_3_field.description == \ - "level_3 column description" - - # check the descriptions in the catalog - node = catalog_data['nodes']['model.test.{}'.format(node_id)] - - level_1_column = node['columns']['level_1'] - assert level_1_column['comment'] == "level_1 column description" - - level_2_column = node['columns']['level_1.level_2'] - assert level_2_column['comment'] == "level_2 column description" - - level_3_column = node['columns']['level_1.level_2.level_3_a'] - assert level_3_column['comment'] == "level_3 column description" - - -class TestPersistDocsColumnMissing(BasePersistDocsTest): - @property - def project_config(self): - return { - 'config-version': 2, - 'models': { - 'test': { - '+persist_docs': { - "columns": True, - }, - } - } - } - - @property - def models(self): - return 'models-column-missing' - - @use_profile('snowflake') - def test_snowflake_missing_column(self): - self.run_dbt() - self.run_dbt(['docs', 'generate']) - with open('target/catalog.json') as fp: - catalog_data = json.load(fp) - assert 'nodes' in catalog_data - - table_node = catalog_data['nodes']['model.test.missing_column'] - table_id_comment = table_node['columns']['ID']['comment'] - assert table_id_comment.startswith('test id column description') - - @use_profile('bigquery') - def test_bigquery_missing_column(self): - self.run_dbt() diff --git a/test/integration/062_defer_state_test/test_defer_state.py b/test/integration/062_defer_state_test/test_defer_state.py index a0aff8fa493..cdd29313e25 100644 --- a/test/integration/062_defer_state_test/test_defer_state.py +++ b/test/integration/062_defer_state_test/test_defer_state.py @@ -35,8 +35,6 @@ def project_config(self): def get_profile(self, adapter_type): if self.other_schema is None: self.other_schema = self.unique_schema() + '_other' - if self.adapter_type == 'snowflake': - self.other_schema = self.other_schema.upper() profile = super().get_profile(adapter_type) default_name = profile['test']['target'] profile['test']['outputs']['otherschema'] = copy.deepcopy(profile['test']['outputs'][default_name]) @@ -119,7 +117,6 @@ def run_defer_iff_not_exists(self): # copy files over from the happy times when we had a good target self.copy_state() - results = self.run_dbt(['seed']) assert len(results) == 1 results = self.run_dbt(['run', '--state', 'state', '--defer']) @@ -128,7 +125,7 @@ def run_defer_iff_not_exists(self): # because the seed now exists in our schema, we shouldn't defer it assert self.other_schema not in results[0].node.compiled_sql assert self.unique_schema() in results[0].node.compiled_sql - + def run_defer_deleted_upstream(self): results = self.run_dbt(['seed']) assert len(results) == 1 @@ -139,7 +136,7 @@ def run_defer_deleted_upstream(self): self.copy_state() self.use_default_project({'source-paths': ['changed_models_missing']}) - # ephemeral_model is now gone. previously this caused a + # ephemeral_model is now gone. previously this caused a # keyerror (dbt#2875), now it should pass self.run_dbt( ['run', '-m', 'view_model', '--state', 'state', '--defer', '--target', 'otherschema'], @@ -166,24 +163,11 @@ def test_postgres_state_changetarget(self): @use_profile('postgres') def test_postgres_state_changedir(self): self.run_switchdirs_defer() - + @use_profile('postgres') def test_postgres_state_defer_iffnotexists(self): self.run_defer_iff_not_exists() - + @use_profile('postgres') def test_postgres_state_defer_deleted_upstream(self): - self.run_defer_deleted_upstream() - - @use_profile('snowflake') - def test_snowflake_state_changetarget(self): - self.run_and_defer() - - @use_profile('redshift') - def test_redshift_state_changetarget(self): - self.run_and_defer() - - @use_profile('bigquery') - def test_bigquery_state_changetarget(self): - self.run_and_defer() - + self.run_defer_deleted_upstream() diff --git a/test/integration/063_relation_name_tests/test_relation_name.py b/test/integration/063_relation_name_tests/test_relation_name.py index cec13e6b795..df81b57f69b 100644 --- a/test/integration/063_relation_name_tests/test_relation_name.py +++ b/test/integration/063_relation_name_tests/test_relation_name.py @@ -1,6 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile from pytest import mark -import dbt.exceptions class TestAdapterDDL(DBTIntegrationTest): @@ -73,7 +72,3 @@ def test_postgres_long_name_passes_when_temp_tables_are_generated(self): ], expect_pass=True, ) - - @use_profile("redshift") - def test_redshift_long_name_succeeds(self): - self.run_dbt(["run"], expect_pass=True) diff --git a/test/integration/064_column_comments_tests/test_column_comments.py b/test/integration/064_column_comments_tests/test_column_comments.py index 4e8924003c8..bd94b642cb6 100644 --- a/test/integration/064_column_comments_tests/test_column_comments.py +++ b/test/integration/064_column_comments_tests/test_column_comments.py @@ -1,8 +1,7 @@ -from test.integration.base import DBTIntegrationTest, use_profile -import os - import json +from test.integration.base import DBTIntegrationTest, use_profile + class TestColumnComment(DBTIntegrationTest): @property @@ -39,18 +38,6 @@ def run_has_comments(self): column_comment = column_node['columns']['2id']['comment'] assert column_comment.startswith('XXX') - @use_profile('postgres') def test_postgres_comments(self): self.run_has_comments() - - @use_profile('redshift') - def test_redshift_comments(self): - self.run_has_comments() - - @use_profile('snowflake') - def test_snowflake_comments(self): - self.run_has_comments() - - - diff --git a/test/integration/067_store_test_failures_tests/test_store_test_failures.py b/test/integration/067_store_test_failures_tests/test_store_test_failures.py index 2949817f6b0..b0ba0875128 100644 --- a/test/integration/067_store_test_failures_tests/test_store_test_failures.py +++ b/test/integration/067_store_test_failures_tests/test_store_test_failures.py @@ -1,7 +1,4 @@ -from test.integration.base import DBTIntegrationTest, FakeArgs, use_profile - -from dbt.task.test import TestTask -from dbt.task.list import ListTask +from test.integration.base import DBTIntegrationTest, use_profile class TestStoreTestFailures(DBTIntegrationTest): @@ -32,10 +29,10 @@ def project_config(self): }, }, } - + def column_type_overrides(self): return {} - + def run_tests_store_one_failure(self): test_audit_schema = self.unique_schema() + "_dbt_test__audit" @@ -58,7 +55,7 @@ def run_tests_store_failures_and_assert(self): # compare test results actual = [(r.status, r.failures) for r in results] expected = [('pass', 0), ('pass', 0), ('pass', 0), ('pass', 0), - ('fail', 2), ('fail', 2), ('fail', 2), ('fail', 10),] + ('fail', 2), ('fail', 2), ('fail', 2), ('fail', 10)] self.assertEqual(sorted(actual), sorted(expected)) # compare test results stored in database @@ -67,11 +64,13 @@ def run_tests_store_failures_and_assert(self): self.assertTablesEqual("unique_problematic_model_id", "expected_unique_problematic_model_id", test_audit_schema) self.assertTablesEqual("accepted_values_problematic_mo_c533ab4ca65c1a9dbf14f79ded49b628", "expected_accepted_values", test_audit_schema) + class PostgresTestStoreTestFailures(TestStoreTestFailures): + @property def schema(self): - return "067" # otherwise too long + truncated - + return "067" # otherwise too long + truncated + def column_type_overrides(self): return { "expected_unique_problematic_model_id": { @@ -85,44 +84,8 @@ def column_type_overrides(self): }, }, } - + @use_profile('postgres') def test__postgres__store_and_assert(self): self.run_tests_store_one_failure() self.run_tests_store_failures_and_assert() - -class RedshiftTestStoreTestFailures(TestStoreTestFailures): - def column_type_overrides(self): - return { - "expected_not_null_problematic_model_id": { - "+column_types": { - "email": "varchar(26)", - "first_name": "varchar(10)", - }, - }, - "expected_unique_problematic_model_id": { - "+column_types": { - "n_records": "bigint", - }, - }, - "expected_accepted_values": { - "+column_types": { - "value_field": "varchar(10)", - "n_records": "bigint", - }, - }, - } - - @use_profile('redshift') - def test__redshift__store_and_assert(self): - self.run_tests_store_failures_and_assert() - -class SnowflakeTestStoreTestFailures(TestStoreTestFailures): - @use_profile('snowflake') - def test__snowflake__store_and_assert(self): - self.run_tests_store_failures_and_assert() - -class BigQueryTestStoreTestFailures(TestStoreTestFailures): - @use_profile('bigquery') - def test__bigquery__store_and_assert(self): - self.run_tests_store_failures_and_assert() diff --git a/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns.sql b/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns.sql index 18d0d5d889a..f9eebdcb852 100644 --- a/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns.sql +++ b/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns.sql @@ -6,7 +6,7 @@ ) }} -{% set string_type = 'string' if target.type == 'bigquery' else 'varchar(10)' %} +{% set string_type = 'varchar(10)' %} WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) diff --git a/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_target.sql b/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_target.sql index 55ed7b2c59b..5ff759d7dab 100644 --- a/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_target.sql +++ b/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_target.sql @@ -2,7 +2,7 @@ config(materialized='table') }} -{% set string_type = 'string' if target.type == 'bigquery' else 'varchar(10)' %} +{% set string_type = 'varchar(10)' %} with source_data as ( diff --git a/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns.sql b/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns.sql index 56a3e3c0fb6..b742c970419 100644 --- a/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns.sql +++ b/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns.sql @@ -9,7 +9,7 @@ WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) -{% set string_type = 'string' if target.type == 'bigquery' else 'varchar(10)' %} +{% set string_type = 'varchar(10)' %} {% if is_incremental() %} diff --git a/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns_target.sql b/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns_target.sql index abffbf7461c..6cdbaba5c0d 100644 --- a/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns_target.sql +++ b/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns_target.sql @@ -8,7 +8,7 @@ with source_data as ( ) -{% set string_type = 'string' if target.type == 'bigquery' else 'varchar(10)' %} +{% set string_type = 'varchar(10)' %} select id ,cast(field1 as {{string_type}}) as field1 diff --git a/test/integration/070_incremental_schema_tests/test_incremental_schema.py b/test/integration/070_incremental_schema_tests/test_incremental_schema.py index d5b0ea4e0f7..9e74ceed9ec 100644 --- a/test/integration/070_incremental_schema_tests/test_incremental_schema.py +++ b/test/integration/070_incremental_schema_tests/test_incremental_schema.py @@ -1,4 +1,4 @@ -from test.integration.base import DBTIntegrationTest, FakeArgs, use_profile +from test.integration.base import DBTIntegrationTest, use_profile class TestIncrementalSchemaChange(DBTIntegrationTest): @@ -18,12 +18,11 @@ def project_config(self): } def list_tests_and_assert(self, include, exclude, expected_tests): - list_args = [ 'ls', '--resource-type', 'test'] + list_args = ['ls', '--resource-type', 'test'] if include: list_args.extend(('--select', include)) if exclude: list_args.extend(('--exclude', exclude)) - listed = self.run_dbt(list_args) print(listed) assert len(listed) == len(expected_tests) @@ -38,13 +37,12 @@ def run_tests_and_assert( run_args = ['run'] if include: run_args.extend(('--models', include)) - results_one = self.run_dbt(run_args) results_two = self.run_dbt(run_args) self.assertEqual(len(results_one), 3) self.assertEqual(len(results_two), 3) - + test_args = ['test'] if include: test_args.extend(('--models', include)) @@ -70,10 +68,10 @@ def run_incremental_ignore(self): 'unique_incremental_ignore_id', 'unique_incremental_ignore_target_id' ] - + self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected, compare_source, compare_target) - + def run_incremental_append_new_columns(self): select = 'model_a incremental_append_new_columns incremental_append_new_columns_target' compare_source = 'incremental_append_new_columns' @@ -87,10 +85,9 @@ def run_incremental_append_new_columns(self): 'unique_incremental_append_new_columns_id', 'unique_incremental_append_new_columns_target_id' ] - self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected, compare_source, compare_target) - + def run_incremental_sync_all_columns(self): select = 'model_a incremental_sync_all_columns incremental_sync_all_columns_target' compare_source = 'incremental_sync_all_columns' @@ -104,17 +101,15 @@ def run_incremental_sync_all_columns(self): 'unique_incremental_sync_all_columns_id', 'unique_incremental_sync_all_columns_target_id' ] - self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected, compare_source, compare_target) - + def run_incremental_fail_on_schema_change(self): select = 'model_a incremental_fail' results_one = self.run_dbt(['run', '--models', select, '--full-refresh']) results_two = self.run_dbt(['run', '--models', select], expect_pass = False) self.assertIn('Compilation Error', results_two[1].message) - - ######################### POSTGRES TESTS ######################### + @use_profile('postgres') def test__postgres__run_incremental_ignore(self): self.run_incremental_ignore() @@ -126,58 +121,7 @@ def test__postgres__run_incremental_append_new_columns(self): @use_profile('postgres') def test__postgres__run_incremental_sync_all_columns(self): self.run_incremental_sync_all_columns() - + @use_profile('postgres') def test__postgres__run_incremental_fail_on_schema_change(self): self.run_incremental_fail_on_schema_change() - - ######################### REDSHIFT TESTS ######################### - @use_profile('redshift') - def test__redshift__run_incremental_ignore(self): - self.run_incremental_ignore() - - @use_profile('redshift') - def test__redshift__run_incremental_append_new_columns(self): - self.run_incremental_append_new_columns() - - @use_profile('redshift') - def test__redshift__run_incremental_sync_all_columns(self): - self.run_incremental_sync_all_columns() - - @use_profile('redshift') - def test__redshift__run_incremental_fail_on_schema_change(self): - self.run_incremental_fail_on_schema_change() - - ######################### SNOWFLAKE TESTS ######################### - @use_profile('snowflake') - def test__snowflake__run_incremental_ignore(self): - self.run_incremental_ignore() - - @use_profile('snowflake') - def test__snowflake__run_incremental_append_new_columns(self): - self.run_incremental_append_new_columns() - - @use_profile('snowflake') - def test__snowflake__run_incremental_sync_all_columns(self): - self.run_incremental_sync_all_columns() - - @use_profile('snowflake') - def test__snowflake__run_incremental_fail_on_schema_change(self): - self.run_incremental_fail_on_schema_change() - - ######################### BIGQUERY TESTS ######################### - @use_profile('bigquery') - def test__bigquery__run_incremental_ignore(self): - self.run_incremental_ignore() - - @use_profile('bigquery') - def test__bigquery__run_incremental_append_new_columns(self): - self.run_incremental_append_new_columns() - - @use_profile('bigquery') - def test__bigquery__run_incremental_sync_all_columns(self): - self.run_incremental_sync_all_columns() - - @use_profile('bigquery') - def test__bigquery__run_incremental_fail_on_schema_change(self): - self.run_incremental_fail_on_schema_change() \ No newline at end of file diff --git a/test/integration/100_rpc_test/TODO b/test/integration/100_rpc_test/TODO new file mode 100644 index 00000000000..1e329f5e68d --- /dev/null +++ b/test/integration/100_rpc_test/TODO @@ -0,0 +1,3 @@ +rm this directory, either: +- move tests to the rpc directory +- ensure that this is covered by tests elsewhere diff --git a/test/integration/100_rpc_test/test_execute_fetch_and_serialize.py b/test/integration/100_rpc_test/test_execute_fetch_and_serialize.py index 4af5cd81da1..e0b31e781c1 100644 --- a/test/integration/100_rpc_test/test_execute_fetch_and_serialize.py +++ b/test/integration/100_rpc_test/test_execute_fetch_and_serialize.py @@ -47,29 +47,31 @@ def assert_all_columns_are_strings(self, table): for value in row: self.assertEqual(type(value), str, f'Found a not-string: {value} in row {row}') - @use_profile('bigquery') + # commenting these out for now, to avoid raising undefined profiles error + + #@use_profile('bigquery') def test__bigquery_fetch_and_serialize(self): self.do_test_file('bigquery.sql') - @use_profile('snowflake') + #@use_profile('snowflake') def test__snowflake_fetch_and_serialize(self): self.do_test_file('snowflake.sql') - @use_profile('redshift') + #@use_profile('redshift') def test__redshift_fetch_and_serialize(self): self.do_test_file('redshift.sql') - @use_profile('bigquery') + #@use_profile('bigquery') def test__bigquery_type_coercion(self): table = self.do_test_file('generic.sql') self.assert_all_columns_are_strings(table) - @use_profile('snowflake') + #@use_profile('snowflake') def test__snowflake_type_coercion(self): table = self.do_test_file('generic.sql') self.assert_all_columns_are_strings(table) - @use_profile('redshift') + #@use_profile('redshift') def test__redshift_type_coercion(self): table = self.do_test_file('generic.sql') self.assert_all_columns_are_strings(table) diff --git a/test/integration/base.py b/test/integration/base.py index bf18d33aaad..9899aed2d64 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -17,7 +17,7 @@ from unittest.mock import patch import dbt.main as dbt -import dbt.flags as flags +from dbt import flags from dbt.deprecations import reset_deprecations from dbt.adapters.factory import get_adapter, reset_adapters, register_adapter from dbt.clients.jinja import template_cache @@ -81,7 +81,7 @@ def __init__(self, kwargs): def _profile_from_test_name(test_name): - adapter_names = ('postgres', 'snowflake', 'redshift', 'bigquery', 'presto') + adapter_names = ('postgres', 'presto') adapters_in_name = sum(x in test_name for x in adapter_names) if adapters_in_name != 1: raise ValueError( @@ -175,28 +175,6 @@ def postgres_profile(self): } } - def redshift_profile(self): - return { - 'config': { - 'send_anonymous_usage_stats': False - }, - 'test': { - 'outputs': { - 'default2': { - 'type': 'redshift', - 'threads': 1, - 'host': os.getenv('REDSHIFT_TEST_HOST'), - 'port': int(os.getenv('REDSHIFT_TEST_PORT')), - 'user': os.getenv('REDSHIFT_TEST_USER'), - 'pass': os.getenv('REDSHIFT_TEST_PASS'), - 'dbname': os.getenv('REDSHIFT_TEST_DBNAME'), - 'schema': self.unique_schema() - } - }, - 'target': 'default2' - } - } - def snowflake_profile(self): return { 'config': { @@ -242,39 +220,6 @@ def snowflake_profile(self): } } - def bigquery_profile(self): - credentials_json_str = os.getenv('BIGQUERY_TEST_SERVICE_ACCOUNT_JSON').replace("'", '') - credentials = json.loads(credentials_json_str) - project_id = credentials.get('project_id') - - return { - 'config': { - 'send_anonymous_usage_stats': False - }, - 'test': { - 'outputs': { - 'default2': { - 'type': 'bigquery', - 'method': 'service-account-json', - 'threads': 1, - 'project': project_id, - 'keyfile_json': credentials, - 'schema': self.unique_schema(), - }, - 'alternate': { - 'type': 'bigquery', - 'method': 'service-account-json', - 'threads': 1, - 'project': project_id, - 'keyfile_json': credentials, - 'schema': self.unique_schema(), - 'execution_project': self.alternative_database, - }, - }, - 'target': 'default2' - } - } - def presto_profile(self): return { 'config': { @@ -323,21 +268,11 @@ def default_database(self): @property def alternative_database(self): - if self.adapter_type == 'bigquery': - return os.environ['BIGQUERY_TEST_ALT_DATABASE'] - elif self.adapter_type == 'snowflake': - return os.environ['SNOWFLAKE_TEST_ALT_DATABASE'] return None def get_profile(self, adapter_type): if adapter_type == 'postgres': return self.postgres_profile() - elif adapter_type == 'snowflake': - return self.snowflake_profile() - elif adapter_type == 'bigquery': - return self.bigquery_profile() - elif adapter_type == 'redshift': - return self.redshift_profile() elif adapter_type == 'presto': return self.presto_profile() else: @@ -508,16 +443,12 @@ def _get_schema_fqn(self, database, schema): return schema_fqn def _create_schema_named(self, database, schema): - if self.adapter_type == 'bigquery': - relation = self.adapter.Relation.create(database=database, schema=schema) - self.adapter.create_schema(relation) - else: - schema_fqn = self._get_schema_fqn(database, schema) - self.run_sql(self.CREATE_SCHEMA_STATEMENT.format(schema_fqn)) - self._created_schemas.add(schema_fqn) + schema_fqn = self._get_schema_fqn(database, schema) + self.run_sql(self.CREATE_SCHEMA_STATEMENT.format(schema_fqn)) + self._created_schemas.add(schema_fqn) def _drop_schema_named(self, database, schema): - if self.adapter_type == 'bigquery' or self.adapter_type == 'presto': + if self.adapter_type == 'presto': relation = self.adapter.Relation.create(database=database, schema=schema) self.adapter.drop_schema(relation) else: @@ -533,7 +464,7 @@ def _create_schemas(self): def _drop_schemas_adapter(self): schema = self.unique_schema() - if self.adapter_type == 'bigquery' or self.adapter_type == 'presto': + if self.adapter_type == 'presto': self._drop_schema_named(self.default_database, schema) if self.setup_alternate_db and self.alternative_database: self._drop_schema_named(self.alternative_database, schema) @@ -544,10 +475,10 @@ def _drop_schemas_sql(self): self._created_schemas.add( self._get_schema_fqn(self.default_database, schema) ) - # on postgres/redshift, this will make you sad + # on postgres, this will make you sad drop_alternative = ( self.setup_alternate_db and - self.adapter_type not in {'postgres', 'redshift'} and + self.adapter_type not in {'postgres'} and self.alternative_database ) if drop_alternative: @@ -562,7 +493,7 @@ def _drop_schemas_sql(self): def _drop_schemas(self): with self.adapter.connection_named('__test'): - if self.adapter_type == 'bigquery' or self.adapter_type == 'presto': + if self.adapter_type == 'presto': self._drop_schemas_adapter() else: self._drop_schemas_sql() @@ -585,7 +516,6 @@ def run_dbt(self, args=None, expect_pass=True, profiles_dir=True): return res - def run_dbt_and_capture(self, *args, **kwargs): try: initial_stdout = log_manager.stdout @@ -641,24 +571,10 @@ def transform_sql(self, query, kwargs=None): kwargs = {} base_kwargs.update(kwargs) - to_return = to_return.format(**base_kwargs) return to_return - def run_sql_bigquery(self, sql, fetch): - """Run an SQL query on a bigquery adapter. No cursors, transactions, - etc. to worry about""" - - do_fetch = fetch != 'None' - _, res = self.adapter.execute(sql, fetch=do_fetch) - - # convert dataframe to matrix-ish repr - if fetch == 'one': - return res[0] - else: - return list(res) - def run_sql_presto(self, sql, fetch, conn): cursor = conn.handle.cursor() try: @@ -711,9 +627,7 @@ def run_sql(self, query, fetch='None', kwargs=None, connection_name=None): with self.get_connection(connection_name) as conn: logger.debug('test connection "{}" executing: {}'.format(conn.name, sql)) - if self.adapter_type == 'bigquery': - return self.run_sql_bigquery(sql, fetch) - elif self.adapter_type == 'presto': + if self.adapter_type == 'presto': return self.run_sql_presto(sql, fetch, conn) else: return self.run_sql_common(sql, fetch, conn) @@ -780,20 +694,9 @@ def get_many_table_columns_information_schema(self, tables, schema, database=Non columns = self.run_sql(sql, fetch='all') return list(map(self.filter_many_columns, columns)) - def get_many_table_columns_bigquery(self, tables, schema, database=None): - result = [] - for table in tables: - relation = self._make_relation(table, schema, database) - columns = self.adapter.get_columns_in_relation(relation) - for col in columns: - result.append((table, col.column, col.dtype, col.char_size)) - return result - def get_many_table_columns(self, tables, schema, database=None): if self.adapter_type == 'snowflake': result = self.get_many_table_columns_snowflake(tables, schema, database) - elif self.adapter_type == 'bigquery': - result = self.get_many_table_columns_bigquery(tables, schema, database) else: result = self.get_many_table_columns_information_schema(tables, schema, database) result.sort(key=lambda x: '{}.{}'.format(x[0], x[1])) @@ -1077,7 +980,6 @@ def assertManyTablesEqual(self, *args): 'num_mismatched nonzero: ' + sql ) - def _assertTableRowCountsEqual(self, relation_a, relation_b): cmp_query = """ with table_a as ( @@ -1095,7 +997,6 @@ def _assertTableRowCountsEqual(self, relation_a, relation_b): """.format(str(relation_a), str(relation_b)) - res = self.run_sql(cmp_query, fetch='one') self.assertEqual(int(res[0]), 0, "Row count of table {} doesn't match row count of table {}. ({} rows different)".format( @@ -1230,14 +1131,6 @@ def __repr__(self): return 'AnyStringWith<{!r}>'.format(self.contains) -def bigquery_rate_limiter(err, *args): - msg = str(err) - if 'too many table update operations for this table' in msg: - time.sleep(1) - return True - return False - - def get_manifest(): path = './target/partial_parse.msgpack' if os.path.exists(path): @@ -1247,4 +1140,3 @@ def get_manifest(): return manifest else: return None - diff --git a/test/unit/test_bigquery_adapter.py b/test/unit/test_bigquery_adapter.py deleted file mode 100644 index 46f38afde40..00000000000 --- a/test/unit/test_bigquery_adapter.py +++ /dev/null @@ -1,1018 +0,0 @@ -import agate -import decimal -import json -import string -import random -import re -import pytest -import unittest -from contextlib import contextmanager -from requests.exceptions import ConnectionError -from unittest.mock import patch, MagicMock, Mock, create_autospec, ANY - -import dbt.dataclass_schema - -import dbt.flags as flags - -from dbt.adapters.bigquery import BigQueryCredentials -from dbt.adapters.bigquery import BigQueryAdapter -from dbt.adapters.bigquery import BigQueryRelation -from dbt.adapters.bigquery import Plugin as BigQueryPlugin -from dbt.adapters.bigquery.connections import BigQueryConnectionManager -from dbt.adapters.bigquery.connections import _sanitize_label, _VALIDATE_LABEL_LENGTH_LIMIT -from dbt.adapters.base.query_headers import MacroQueryStringSetter -from dbt.clients import agate_helper -import dbt.exceptions -from dbt.logger import GLOBAL_LOGGER as logger # noqa -from dbt.context.providers import RuntimeConfigObject - -import google.cloud.bigquery - -from .utils import config_from_parts_or_dicts, inject_adapter, TestAdapterConversions - - -def _bq_conn(): - conn = MagicMock() - conn.get.side_effect = lambda x: 'bigquery' if x == 'type' else None - return conn - - -class BaseTestBigQueryAdapter(unittest.TestCase): - - def setUp(self): - self.raw_profile = { - 'outputs': { - 'oauth': { - 'type': 'bigquery', - 'method': 'oauth', - 'project': 'dbt-unit-000000', - 'schema': 'dummy_schema', - 'threads': 1, - }, - 'service_account': { - 'type': 'bigquery', - 'method': 'service-account', - 'project': 'dbt-unit-000000', - 'schema': 'dummy_schema', - 'keyfile': '/tmp/dummy-service-account.json', - 'threads': 1, - }, - 'loc': { - 'type': 'bigquery', - 'method': 'oauth', - 'project': 'dbt-unit-000000', - 'schema': 'dummy_schema', - 'threads': 1, - 'location': 'Luna Station', - 'priority': 'batch', - 'maximum_bytes_billed': 0, - }, - 'impersonate': { - 'type': 'bigquery', - 'method': 'oauth', - 'project': 'dbt-unit-000000', - 'schema': 'dummy_schema', - 'threads': 1, - 'impersonate_service_account': 'dummyaccount@dbt.iam.gserviceaccount.com' - }, - 'oauth-credentials-token': { - 'type': 'bigquery', - 'method': 'oauth-secrets', - 'token': 'abc', - 'project': 'dbt-unit-000000', - 'schema': 'dummy_schema', - 'threads': 1, - 'location': 'Luna Station', - 'priority': 'batch', - 'maximum_bytes_billed': 0, - }, - 'oauth-credentials': { - 'type': 'bigquery', - 'method': 'oauth-secrets', - 'client_id': 'abc', - 'client_secret': 'def', - 'refresh_token': 'ghi', - 'token_uri': 'jkl', - 'project': 'dbt-unit-000000', - 'schema': 'dummy_schema', - 'threads': 1, - 'location': 'Luna Station', - 'priority': 'batch', - 'maximum_bytes_billed': 0, - }, - 'oauth-no-project': { - 'type': 'bigquery', - 'method': 'oauth', - 'schema': 'dummy_schema', - 'threads': 1, - 'location': 'Solar Station', - }, - }, - 'target': 'oauth', - } - - self.project_cfg = { - 'name': 'X', - 'version': '0.1', - 'project-root': '/tmp/dbt/does-not-exist', - 'profile': 'default', - 'config-version': 2, - } - self.qh_patch = None - - def tearDown(self): - if self.qh_patch: - self.qh_patch.stop() - super().tearDown() - - def get_adapter(self, target): - project = self.project_cfg.copy() - profile = self.raw_profile.copy() - profile['target'] = target - - config = config_from_parts_or_dicts( - project=project, - profile=profile, - ) - adapter = BigQueryAdapter(config) - - adapter.connections.query_header = MacroQueryStringSetter(config, MagicMock(macros={})) - - self.qh_patch = patch.object(adapter.connections.query_header, 'add') - self.mock_query_header_add = self.qh_patch.start() - self.mock_query_header_add.side_effect = lambda q: '/* dbt */\n{}'.format(q) - - inject_adapter(adapter, BigQueryPlugin) - return adapter - - -class TestBigQueryAdapterAcquire(BaseTestBigQueryAdapter): - @patch('dbt.adapters.bigquery.connections.get_bigquery_defaults', return_value=('credentials', 'project_id')) - @patch('dbt.adapters.bigquery.BigQueryConnectionManager.open', return_value=_bq_conn()) - def test_acquire_connection_oauth_no_project_validations(self, mock_open_connection, mock_get_bigquery_defaults): - adapter = self.get_adapter('oauth-no-project') - mock_get_bigquery_defaults.assert_called_once() - try: - connection = adapter.acquire_connection('dummy') - self.assertEqual(connection.type, 'bigquery') - - except dbt.exceptions.ValidationException as e: - self.fail('got ValidationException: {}'.format(str(e))) - - except BaseException as e: - raise - - mock_open_connection.assert_not_called() - connection.handle - mock_open_connection.assert_called_once() - - @patch('dbt.adapters.bigquery.BigQueryConnectionManager.open', return_value=_bq_conn()) - def test_acquire_connection_oauth_validations(self, mock_open_connection): - adapter = self.get_adapter('oauth') - try: - connection = adapter.acquire_connection('dummy') - self.assertEqual(connection.type, 'bigquery') - - except dbt.exceptions.ValidationException as e: - self.fail('got ValidationException: {}'.format(str(e))) - - except BaseException as e: - raise - - mock_open_connection.assert_not_called() - connection.handle - mock_open_connection.assert_called_once() - - @patch('dbt.adapters.bigquery.BigQueryConnectionManager.open', return_value=_bq_conn()) - def test_acquire_connection_service_account_validations(self, mock_open_connection): - adapter = self.get_adapter('service_account') - try: - connection = adapter.acquire_connection('dummy') - self.assertEqual(connection.type, 'bigquery') - - except dbt.exceptions.ValidationException as e: - self.fail('got ValidationException: {}'.format(str(e))) - - except BaseException as e: - raise - - mock_open_connection.assert_not_called() - connection.handle - mock_open_connection.assert_called_once() - - @patch('dbt.adapters.bigquery.BigQueryConnectionManager.open', return_value=_bq_conn()) - def test_acquire_connection_oauth_token_validations(self, mock_open_connection): - adapter = self.get_adapter('oauth-credentials-token') - try: - connection = adapter.acquire_connection('dummy') - self.assertEqual(connection.type, 'bigquery') - - except dbt.exceptions.ValidationException as e: - self.fail('got ValidationException: {}'.format(str(e))) - - except BaseException as e: - raise - - mock_open_connection.assert_not_called() - connection.handle - mock_open_connection.assert_called_once() - - @patch('dbt.adapters.bigquery.BigQueryConnectionManager.open', return_value=_bq_conn()) - def test_acquire_connection_oauth_credentials_validations(self, mock_open_connection): - adapter = self.get_adapter('oauth-credentials') - try: - connection = adapter.acquire_connection('dummy') - self.assertEqual(connection.type, 'bigquery') - - except dbt.exceptions.ValidationException as e: - self.fail('got ValidationException: {}'.format(str(e))) - - except BaseException as e: - raise - - mock_open_connection.assert_not_called() - connection.handle - mock_open_connection.assert_called_once() - - @patch('dbt.adapters.bigquery.BigQueryConnectionManager.open', return_value=_bq_conn()) - def test_acquire_connection_impersonated_service_account_validations(self, mock_open_connection): - adapter = self.get_adapter('impersonate') - try: - connection = adapter.acquire_connection('dummy') - self.assertEqual(connection.type, 'bigquery') - - except dbt.exceptions.ValidationException as e: - self.fail('got ValidationException: {}'.format(str(e))) - - except BaseException as e: - raise - - mock_open_connection.assert_not_called() - connection.handle - mock_open_connection.assert_called_once() - - @patch('dbt.adapters.bigquery.BigQueryConnectionManager.open', return_value=_bq_conn()) - def test_acquire_connection_priority(self, mock_open_connection): - adapter = self.get_adapter('loc') - try: - connection = adapter.acquire_connection('dummy') - self.assertEqual(connection.type, 'bigquery') - self.assertEqual(connection.credentials.priority, 'batch') - - except dbt.exceptions.ValidationException as e: - self.fail('got ValidationException: {}'.format(str(e))) - - mock_open_connection.assert_not_called() - connection.handle - mock_open_connection.assert_called_once() - - @patch('dbt.adapters.bigquery.BigQueryConnectionManager.open', return_value=_bq_conn()) - def test_acquire_connection_maximum_bytes_billed(self, mock_open_connection): - adapter = self.get_adapter('loc') - try: - connection = adapter.acquire_connection('dummy') - self.assertEqual(connection.type, 'bigquery') - self.assertEqual(connection.credentials.maximum_bytes_billed, 0) - - except dbt.exceptions.ValidationException as e: - self.fail('got ValidationException: {}'.format(str(e))) - - mock_open_connection.assert_not_called() - connection.handle - mock_open_connection.assert_called_once() - - def test_cancel_open_connections_empty(self): - adapter = self.get_adapter('oauth') - self.assertEqual(adapter.cancel_open_connections(), None) - - def test_cancel_open_connections_master(self): - adapter = self.get_adapter('oauth') - adapter.connections.thread_connections[0] = object() - self.assertEqual(adapter.cancel_open_connections(), None) - - def test_cancel_open_connections_single(self): - adapter = self.get_adapter('oauth') - adapter.connections.thread_connections.update({ - 0: object(), - 1: object(), - }) - # actually does nothing - self.assertEqual(adapter.cancel_open_connections(), None) - - @patch('dbt.adapters.bigquery.impl.google.auth.default') - @patch('dbt.adapters.bigquery.impl.google.cloud.bigquery') - def test_location_user_agent(self, mock_bq, mock_auth_default): - creds = MagicMock() - mock_auth_default.return_value = (creds, MagicMock()) - adapter = self.get_adapter('loc') - - connection = adapter.acquire_connection('dummy') - mock_client = mock_bq.Client - - mock_client.assert_not_called() - connection.handle - mock_client.assert_called_once_with('dbt-unit-000000', creds, - location='Luna Station', - client_info=HasUserAgent()) - - -class HasUserAgent: - PAT = re.compile(r'dbt-\d+\.\d+\.\d+((a|b|rc)\d+)?') - - def __eq__(self, other): - compare = getattr(other, 'user_agent', '') - return bool(self.PAT.match(compare)) - - -class TestConnectionNamePassthrough(BaseTestBigQueryAdapter): - - def setUp(self): - super().setUp() - self._conn_patch = patch.object(BigQueryAdapter, 'ConnectionManager') - self.conn_manager_cls = self._conn_patch.start() - - self._relation_patch = patch.object(BigQueryAdapter, 'Relation') - self.relation_cls = self._relation_patch.start() - - self.mock_connection_manager = self.conn_manager_cls.return_value - self.conn_manager_cls.TYPE = 'bigquery' - self.relation_cls.get_default_quote_policy.side_effect = BigQueryRelation.get_default_quote_policy - - self.adapter = self.get_adapter('oauth') - - def tearDown(self): - super().tearDown() - self._conn_patch.stop() - self._relation_patch.stop() - - def test_get_relation(self): - self.adapter.get_relation('db', 'schema', 'my_model') - self.mock_connection_manager.get_bq_table.assert_called_once_with('db', 'schema', 'my_model') - - def test_create_schema(self): - relation = BigQueryRelation.create(database='db', schema='schema') - self.adapter.create_schema(relation) - self.mock_connection_manager.create_dataset.assert_called_once_with('db', 'schema') - - @patch.object(BigQueryAdapter, 'check_schema_exists') - def test_drop_schema(self, mock_check_schema): - mock_check_schema.return_value = True - relation = BigQueryRelation.create(database='db', schema='schema') - self.adapter.drop_schema(relation) - self.mock_connection_manager.drop_dataset.assert_called_once_with('db', 'schema') - - def test_get_columns_in_relation(self): - self.mock_connection_manager.get_bq_table.side_effect = ValueError - self.adapter.get_columns_in_relation( - MagicMock(database='db', schema='schema', identifier='ident'), - ) - self.mock_connection_manager.get_bq_table.assert_called_once_with( - database='db', schema='schema', identifier='ident' - ) - - -class TestBigQueryRelation(unittest.TestCase): - def setUp(self): - pass - - def test_view_temp_relation(self): - kwargs = { - 'type': None, - 'path': { - 'database': 'test-project', - 'schema': 'test_schema', - 'identifier': 'my_view' - }, - 'quote_policy': { - 'identifier': False - } - } - BigQueryRelation.validate(kwargs) - - def test_view_relation(self): - kwargs = { - 'type': 'view', - 'path': { - 'database': 'test-project', - 'schema': 'test_schema', - 'identifier': 'my_view' - }, - 'quote_policy': { - 'identifier': True, - 'schema': True - } - } - BigQueryRelation.validate(kwargs) - - def test_table_relation(self): - kwargs = { - 'type': 'table', - 'path': { - 'database': 'test-project', - 'schema': 'test_schema', - 'identifier': 'generic_table' - }, - 'quote_policy': { - 'identifier': True, - 'schema': True - } - } - BigQueryRelation.validate(kwargs) - - def test_external_source_relation(self): - kwargs = { - 'type': 'external', - 'path': { - 'database': 'test-project', - 'schema': 'test_schema', - 'identifier': 'sheet' - }, - 'quote_policy': { - 'identifier': True, - 'schema': True - } - } - BigQueryRelation.validate(kwargs) - - def test_invalid_relation(self): - kwargs = { - 'type': 'invalid-type', - 'path': { - 'database': 'test-project', - 'schema': 'test_schema', - 'identifier': 'my_invalid_id' - }, - 'quote_policy': { - 'identifier': False, - 'schema': True - } - } - with self.assertRaises(dbt.dataclass_schema.ValidationError): - BigQueryRelation.validate(kwargs) - - -class TestBigQueryInformationSchema(unittest.TestCase): - def setUp(self): - pass - - def test_replace(self): - - kwargs = { - 'type': None, - 'path': { - 'database': 'test-project', - 'schema': 'test_schema', - 'identifier': 'my_view' - }, - # test for #2188 - 'quote_policy': { - 'database': False - }, - 'include_policy': { - 'database': True, - 'schema': True, - 'identifier': True, - } - } - BigQueryRelation.validate(kwargs) - relation = BigQueryRelation.from_dict(kwargs) - info_schema = relation.information_schema() - - tables_schema = info_schema.replace(information_schema_view='__TABLES__') - assert tables_schema.information_schema_view == '__TABLES__' - assert tables_schema.include_policy.schema is True - assert tables_schema.include_policy.identifier is False - assert tables_schema.include_policy.database is True - assert tables_schema.quote_policy.schema is True - assert tables_schema.quote_policy.identifier is False - assert tables_schema.quote_policy.database is False - - schemata_schema = info_schema.replace(information_schema_view='SCHEMATA') - assert schemata_schema.information_schema_view == 'SCHEMATA' - assert schemata_schema.include_policy.schema is False - assert schemata_schema.include_policy.identifier is True - assert schemata_schema.include_policy.database is True - assert schemata_schema.quote_policy.schema is True - assert schemata_schema.quote_policy.identifier is False - assert schemata_schema.quote_policy.database is False - - other_schema = info_schema.replace(information_schema_view='SOMETHING_ELSE') - assert other_schema.information_schema_view == 'SOMETHING_ELSE' - assert other_schema.include_policy.schema is True - assert other_schema.include_policy.identifier is True - assert other_schema.include_policy.database is True - assert other_schema.quote_policy.schema is True - assert other_schema.quote_policy.identifier is False - assert other_schema.quote_policy.database is False - - -class TestBigQueryConnectionManager(unittest.TestCase): - - def setUp(self): - credentials = Mock(BigQueryCredentials) - profile = Mock(query_comment=None, credentials=credentials) - self.connections = BigQueryConnectionManager(profile=profile) - self.mock_client = Mock( - dbt.adapters.bigquery.impl.google.cloud.bigquery.Client) - self.mock_connection = MagicMock() - - self.mock_connection.handle = self.mock_client - - self.connections.get_thread_connection = lambda: self.mock_connection - - @patch( - 'dbt.adapters.bigquery.connections._is_retryable', return_value=True) - def test_retry_and_handle(self, is_retryable): - self.connections.DEFAULT_MAXIMUM_DELAY = 2.0 - - @contextmanager - def dummy_handler(msg): - yield - - self.connections.exception_handler = dummy_handler - - class DummyException(Exception): - """Count how many times this exception is raised""" - count = 0 - - def __init__(self): - DummyException.count += 1 - - def raiseDummyException(): - raise DummyException() - - with self.assertRaises(DummyException): - self.connections._retry_and_handle( - "some sql", Mock(credentials=Mock(retries=8)), - raiseDummyException) - self.assertEqual(DummyException.count, 9) - - @patch( - 'dbt.adapters.bigquery.connections._is_retryable', return_value=True) - def test_retry_connection_reset(self, is_retryable): - self.connections.open = MagicMock() - self.connections.close = MagicMock() - self.connections.DEFAULT_MAXIMUM_DELAY = 2.0 - - @contextmanager - def dummy_handler(msg): - yield - - self.connections.exception_handler = dummy_handler - - def raiseConnectionResetError(): - raise ConnectionResetError("Connection broke") - - mock_conn = Mock(credentials=Mock(retries=1)) - with self.assertRaises(ConnectionResetError): - self.connections._retry_and_handle( - "some sql", mock_conn, - raiseConnectionResetError) - self.connections.close.assert_called_once_with(mock_conn) - self.connections.open.assert_called_once_with(mock_conn) - - def test_is_retryable(self): - _is_retryable = dbt.adapters.bigquery.connections._is_retryable - exceptions = dbt.adapters.bigquery.impl.google.cloud.exceptions - internal_server_error = exceptions.InternalServerError('code broke') - bad_request_error = exceptions.BadRequest('code broke') - connection_error = ConnectionError('code broke') - client_error = exceptions.ClientError('bad code') - rate_limit_error = exceptions.Forbidden("code broke", errors=[{"reason": "rateLimitExceeded"}]) - - self.assertTrue(_is_retryable(internal_server_error)) - self.assertTrue(_is_retryable(bad_request_error)) - self.assertTrue(_is_retryable(connection_error)) - self.assertFalse(_is_retryable(client_error)) - self.assertTrue(_is_retryable(rate_limit_error)) - - def test_drop_dataset(self): - mock_table = Mock() - mock_table.reference = 'table1' - - self.mock_client.list_tables.return_value = [mock_table] - - self.connections.drop_dataset('project', 'dataset') - - self.mock_client.list_tables.assert_not_called() - self.mock_client.delete_table.assert_not_called() - self.mock_client.delete_dataset.assert_called_once() - - @patch('dbt.adapters.bigquery.impl.google.cloud.bigquery') - def test_query_and_results(self, mock_bq): - self.connections.get_timeout = lambda x: 100.0 - - self.connections._query_and_results( - self.mock_client, 'sql', self.mock_connection, - {'description': 'blah'}) - - mock_bq.QueryJobConfig.assert_called_once() - self.mock_client.query.assert_called_once_with( - 'sql', job_config=mock_bq.QueryJobConfig()) - - def test_copy_bq_table_appends(self): - self._copy_table( - write_disposition=dbt.adapters.bigquery.impl.WRITE_APPEND) - args, kwargs = self.mock_client.copy_table.call_args - self.mock_client.copy_table.assert_called_once_with( - [self._table_ref('project', 'dataset', 'table1', None)], - self._table_ref('project', 'dataset', 'table2', None), - job_config=ANY) - args, kwargs = self.mock_client.copy_table.call_args - self.assertEqual( - kwargs['job_config'].write_disposition, - dbt.adapters.bigquery.impl.WRITE_APPEND) - - def test_copy_bq_table_truncates(self): - self._copy_table( - write_disposition=dbt.adapters.bigquery.impl.WRITE_TRUNCATE) - args, kwargs = self.mock_client.copy_table.call_args - self.mock_client.copy_table.assert_called_once_with( - [self._table_ref('project', 'dataset', 'table1', None)], - self._table_ref('project', 'dataset', 'table2', None), - job_config=ANY) - args, kwargs = self.mock_client.copy_table.call_args - self.assertEqual( - kwargs['job_config'].write_disposition, - dbt.adapters.bigquery.impl.WRITE_TRUNCATE) - - def test_job_labels_valid_json(self): - expected = {"key": "value"} - labels = self.connections._labels_from_query_comment(json.dumps(expected)) - self.assertEqual(labels, expected) - - def test_job_labels_invalid_json(self): - labels = self.connections._labels_from_query_comment("not json") - self.assertEqual(labels, {"query_comment": "not_json"}) - - def _table_ref(self, proj, ds, table, conn): - return google.cloud.bigquery.table.TableReference.from_string( - '{}.{}.{}'.format(proj, ds, table)) - - def _copy_table(self, write_disposition): - self.connections.table_ref = self._table_ref - source = BigQueryRelation.create( - database='project', schema='dataset', identifier='table1') - destination = BigQueryRelation.create( - database='project', schema='dataset', identifier='table2') - self.connections.copy_bq_table(source, destination, write_disposition) - - -class TestBigQueryAdapter(BaseTestBigQueryAdapter): - - def test_copy_table_materialization_table(self): - adapter = self.get_adapter('oauth') - adapter.connections = MagicMock() - adapter.copy_table('source', 'destination', 'table') - adapter.connections.copy_bq_table.assert_called_once_with( - 'source', 'destination', - dbt.adapters.bigquery.impl.WRITE_TRUNCATE) - - def test_copy_table_materialization_incremental(self): - adapter = self.get_adapter('oauth') - adapter.connections = MagicMock() - adapter.copy_table('source', 'destination', 'incremental') - adapter.connections.copy_bq_table.assert_called_once_with( - 'source', 'destination', - dbt.adapters.bigquery.impl.WRITE_APPEND) - - def test_parse_partition_by(self): - adapter = self.get_adapter('oauth') - - with self.assertRaises(dbt.exceptions.CompilationException): - adapter.parse_partition_by("date(ts)") - - with self.assertRaises(dbt.exceptions.CompilationException): - adapter.parse_partition_by("ts") - - self.assertEqual( - adapter.parse_partition_by({ - "field": "ts", - }).to_dict(omit_none=True), { - "field": "ts", - "data_type": "date", - "granularity": "day" - } - ) - - self.assertEqual( - adapter.parse_partition_by({ - "field": "ts", - "data_type": "date", - }).to_dict(omit_none=True), { - "field": "ts", - "data_type": "date", - "granularity": "day" - } - ) - - self.assertEqual( - adapter.parse_partition_by({ - "field": "ts", - "data_type": "date", - "granularity": "MONTH" - - }).to_dict(omit_none=True), { - "field": "ts", - "data_type": "date", - "granularity": "MONTH" - } - ) - - self.assertEqual( - adapter.parse_partition_by({ - "field": "ts", - "data_type": "date", - "granularity": "YEAR" - - }).to_dict(omit_none=True), { - "field": "ts", - "data_type": "date", - "granularity": "YEAR" - } - ) - - self.assertEqual( - adapter.parse_partition_by({ - "field": "ts", - "data_type": "timestamp", - "granularity": "HOUR" - - }).to_dict(omit_none=True), { - "field": "ts", - "data_type": "timestamp", - "granularity": "HOUR" - } - ) - - self.assertEqual( - adapter.parse_partition_by({ - "field": "ts", - "data_type": "timestamp", - "granularity": "MONTH" - - }).to_dict(omit_none=True - ), { - "field": "ts", - "data_type": "timestamp", - "granularity": "MONTH" - } - ) - - self.assertEqual( - adapter.parse_partition_by({ - "field": "ts", - "data_type": "timestamp", - "granularity": "YEAR" - - }).to_dict(omit_none=True), { - "field": "ts", - "data_type": "timestamp", - "granularity": "YEAR" - } - ) - - self.assertEqual( - adapter.parse_partition_by({ - "field": "ts", - "data_type": "datetime", - "granularity": "HOUR" - - }).to_dict(omit_none=True), { - "field": "ts", - "data_type": "datetime", - "granularity": "HOUR" - } - ) - - self.assertEqual( - adapter.parse_partition_by({ - "field": "ts", - "data_type": "datetime", - "granularity": "MONTH" - - }).to_dict(omit_none=True), { - "field": "ts", - "data_type": "datetime", - "granularity": "MONTH" - } - ) - - self.assertEqual( - adapter.parse_partition_by({ - "field": "ts", - "data_type": "datetime", - "granularity": "YEAR" - - }).to_dict(omit_none=True), { - "field": "ts", - "data_type": "datetime", - "granularity": "YEAR" - } - ) - - # Invalid, should raise an error - with self.assertRaises(dbt.exceptions.CompilationException): - adapter.parse_partition_by({}) - - # passthrough - self.assertEqual( - adapter.parse_partition_by({ - "field": "id", - "data_type": "int64", - "range": { - "start": 1, - "end": 100, - "interval": 20 - } - }).to_dict(omit_none=True - ), { - "field": "id", - "data_type": "int64", - "granularity": "day", - "range": { - "start": 1, - "end": 100, - "interval": 20 - } - } - ) - - def test_hours_to_expiration(self): - adapter = self.get_adapter('oauth') - mock_config = create_autospec( - RuntimeConfigObject) - config = {'hours_to_expiration': 4} - mock_config.get.side_effect = lambda name: config.get(name) - - expected = { - 'expiration_timestamp': 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 4 hour)', - } - actual = adapter.get_table_options(mock_config, node={}, temporary=False) - self.assertEqual(expected, actual) - - - def test_hours_to_expiration_temporary(self): - adapter = self.get_adapter('oauth') - mock_config = create_autospec( - RuntimeConfigObject) - config={'hours_to_expiration': 4} - mock_config.get.side_effect = lambda name: config.get(name) - - expected = { - 'expiration_timestamp': ( - 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)'), - } - actual = adapter.get_table_options(mock_config, node={}, temporary=True) - self.assertEqual(expected, actual) - - def test_table_kms_key_name(self): - adapter = self.get_adapter('oauth') - mock_config = create_autospec( - RuntimeConfigObject) - config={'kms_key_name': 'some_key'} - mock_config.get.side_effect = lambda name: config.get(name) - - expected = { - 'kms_key_name': "'some_key'" - } - actual = adapter.get_table_options(mock_config, node={}, temporary=False) - self.assertEqual(expected, actual) - - - def test_view_kms_key_name(self): - adapter = self.get_adapter('oauth') - mock_config = create_autospec( - RuntimeConfigObject) - config={'kms_key_name': 'some_key'} - mock_config.get.side_effect = lambda name: config.get(name) - - expected = {} - actual = adapter.get_view_options(mock_config, node={}) - self.assertEqual(expected, actual) - - - -class TestBigQueryFilterCatalog(unittest.TestCase): - def test__catalog_filter_table(self): - manifest = MagicMock() - manifest.get_used_schemas.return_value = [['a', 'B'], ['a', '1234']] - column_names = ['table_name', 'table_database', 'table_schema', 'something'] - rows = [ - ['foo', 'a', 'b', '1234'], # include - ['foo', 'a', '1234', '1234'], # include, w/ table schema as str - ['foo', 'c', 'B', '1234'], # skip - ['1234', 'A', 'B', '1234'], # include, w/ table name as str - ] - table = agate.Table( - rows, column_names, agate_helper.DEFAULT_TYPE_TESTER - ) - - result = BigQueryAdapter._catalog_filter_table(table, manifest) - assert len(result) == 3 - for row in result.rows: - assert isinstance(row['table_schema'], str) - assert isinstance(row['table_database'], str) - assert isinstance(row['table_name'], str) - assert isinstance(row['something'], decimal.Decimal) - - -class TestBigQueryAdapterConversions(TestAdapterConversions): - def test_convert_text_type(self): - rows = [ - ['', 'a1', 'stringval1'], - ['', 'a2', 'stringvalasdfasdfasdfa'], - ['', 'a3', 'stringval3'], - ] - agate_table = self._make_table_of(rows, agate.Text) - expected = ['string', 'string', 'string'] - for col_idx, expect in enumerate(expected): - assert BigQueryAdapter.convert_text_type(agate_table, col_idx) == expect - - def test_convert_number_type(self): - rows = [ - ['', '23.98', '-1'], - ['', '12.78', '-2'], - ['', '79.41', '-3'], - ] - agate_table = self._make_table_of(rows, agate.Number) - expected = ['int64', 'float64', 'int64'] - for col_idx, expect in enumerate(expected): - assert BigQueryAdapter.convert_number_type(agate_table, col_idx) == expect - - def test_convert_boolean_type(self): - rows = [ - ['', 'false', 'true'], - ['', 'false', 'false'], - ['', 'false', 'true'], - ] - agate_table = self._make_table_of(rows, agate.Boolean) - expected = ['bool', 'bool', 'bool'] - for col_idx, expect in enumerate(expected): - assert BigQueryAdapter.convert_boolean_type(agate_table, col_idx) == expect - - def test_convert_datetime_type(self): - rows = [ - ['', '20190101T01:01:01Z', '2019-01-01 01:01:01'], - ['', '20190102T01:01:01Z', '2019-01-01 01:01:01'], - ['', '20190103T01:01:01Z', '2019-01-01 01:01:01'], - ] - agate_table = self._make_table_of(rows, [agate.DateTime, agate_helper.ISODateTime, agate.DateTime]) - expected = ['datetime', 'datetime', 'datetime'] - for col_idx, expect in enumerate(expected): - assert BigQueryAdapter.convert_datetime_type(agate_table, col_idx) == expect - - def test_convert_date_type(self): - rows = [ - ['', '2019-01-01', '2019-01-04'], - ['', '2019-01-02', '2019-01-04'], - ['', '2019-01-03', '2019-01-04'], - ] - agate_table = self._make_table_of(rows, agate.Date) - expected = ['date', 'date', 'date'] - for col_idx, expect in enumerate(expected): - assert BigQueryAdapter.convert_date_type(agate_table, col_idx) == expect - - def test_convert_time_type(self): - # dbt's default type testers actually don't have a TimeDelta at all. - agate.TimeDelta - rows = [ - ['', '120s', '10s'], - ['', '3m', '11s'], - ['', '1h', '12s'], - ] - agate_table = self._make_table_of(rows, agate.TimeDelta) - expected = ['time', 'time', 'time'] - for col_idx, expect in enumerate(expected): - assert BigQueryAdapter.convert_time_type(agate_table, col_idx) == expect - - -@pytest.mark.parametrize( - ["input", "output"], - [ - ("ABC", "abc"), - ("a c", "a_c"), - ("a ", "a"), - ], -) -def test_sanitize_label(input, output): - assert _sanitize_label(input) == output - - -@pytest.mark.parametrize( - "label_length", - [64, 65, 100], -) -def test_sanitize_label_length(label_length): - random_string = "".join( - random.choice(string.ascii_uppercase + string.digits) - for i in range(label_length) - ) - test_error_msg = ( - f"Job label length {label_length} is greater than length limit: " - f"{_VALIDATE_LABEL_LENGTH_LIMIT}\n" - f"Current sanitized label: {random_string.lower()}" - ) - with pytest.raises(dbt.exceptions.RuntimeException) as error_info: - _sanitize_label(random_string) - assert error_info.value.args[0] == test_error_msg diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 80d6099d828..1005b4e7e83 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -15,7 +15,6 @@ from dbt import flags from dbt.adapters.factory import load_plugin from dbt.adapters.postgres import PostgresCredentials -from dbt.adapters.redshift import RedshiftCredentials from dbt.context.base import generate_base_context from dbt.contracts.connection import QueryComment, DEFAULT_QUERY_COMMENT from dbt.contracts.project import PackageConfig, LocalPackage, GitPackage @@ -37,6 +36,7 @@ def temp_cd(path): finally: os.chdir(current_path) + @contextmanager def raises_nothing(): yield @@ -127,15 +127,6 @@ def setUp(self): 'schema': 'postgres-schema', 'threads': 7, }, - 'redshift': { - 'type': 'redshift', - 'host': 'redshift-db-hostname', - 'port': 5555, - 'user': 'db_user', - 'pass': 'db_pass', - 'dbname': 'redshift-db-name', - 'schema': 'redshift-schema', - }, 'with-vars': { 'type': "{{ env_var('env_value_type') }}", 'host': "{{ env_var('env_value_host') }}", @@ -342,7 +333,6 @@ def test_profile_invalid_target(self): self.assertIn('nope', str(exc.exception)) self.assertIn('- postgres', str(exc.exception)) - self.assertIn('- redshift', str(exc.exception)) self.assertIn('- with-vars', str(exc.exception)) def test_no_outputs(self): @@ -454,28 +444,6 @@ def test_profile_override(self): self.assertEqual(profile.credentials.database, 'other-postgres-db-name') self.assertEqual(profile, from_raw) - def test_target_override(self): - self.args.target = 'redshift' - profile = self.from_args() - from_raw = self.from_raw_profile_info( - target_override='redshift' - ) - - self.assertEqual(profile.profile_name, 'default') - self.assertEqual(profile.target_name, 'redshift') - self.assertEqual(profile.threads, 1) - self.assertTrue(profile.user_config.send_anonymous_usage_stats) - self.assertIsNone(profile.user_config.use_colors) - self.assertTrue(isinstance(profile.credentials, RedshiftCredentials)) - self.assertEqual(profile.credentials.type, 'redshift') - self.assertEqual(profile.credentials.host, 'redshift-db-hostname') - self.assertEqual(profile.credentials.port, 5555) - self.assertEqual(profile.credentials.user, 'db_user') - self.assertEqual(profile.credentials.password, 'db_pass') - self.assertEqual(profile.credentials.schema, 'redshift-schema') - self.assertEqual(profile.credentials.database, 'redshift-db-name') - self.assertEqual(profile, from_raw) - def test_env_vars(self): self.args.target = 'with-vars' with mock.patch.dict(os.environ, self.env_override): diff --git a/test/unit/test_context.py b/test/unit/test_context.py index 23473c989d4..577840acf37 100644 --- a/test/unit/test_context.py +++ b/test/unit/test_context.py @@ -6,8 +6,7 @@ import pytest -# make sure 'redshift' is available -from dbt.adapters import postgres, redshift +from dbt.adapters import postgres from dbt.adapters import factory from dbt.adapters.base import AdapterConfig from dbt.clients.jinja import MacroStack @@ -226,22 +225,6 @@ def assert_has_keys( MAYBE_KEYS = frozenset({'debug'}) -PROFILE_DATA = { - 'target': 'test', - 'quoting': {}, - 'outputs': { - 'test': { - 'type': 'redshift', - 'host': 'localhost', - 'schema': 'analytics', - 'user': 'test', - 'pass': 'test', - 'dbname': 'test', - 'port': 1, - } - }, -} - POSTGRES_PROFILE_DATA = { 'target': 'test', 'quoting': {}, @@ -306,12 +289,6 @@ def test_base_context(): assert_has_keys(REQUIRED_BASE_KEYS, MAYBE_KEYS, ctx) -def test_target_context(): - profile = profile_from_dict(PROFILE_DATA, 'test') - ctx = target.generate_target_context(profile, {}) - assert_has_keys(REQUIRED_TARGET_KEYS, MAYBE_KEYS, ctx) - - def mock_macro(name, package_name): macro = mock.MagicMock( __class__=ParsedMacro, @@ -380,41 +357,14 @@ def get_include_paths(): yield patch -@pytest.fixture -def config(): - return config_from_parts_or_dicts(PROJECT_DATA, PROFILE_DATA) - @pytest.fixture def config_postgres(): return config_from_parts_or_dicts(PROJECT_DATA, POSTGRES_PROFILE_DATA) -@pytest.fixture -def manifest_fx(config): - return mock_manifest(config) - - -@pytest.fixture -def manifest_extended(manifest_fx): - dbt_macro = mock_macro('default__some_macro', 'dbt') - # same namespace, same name, different pkg! - rs_macro = mock_macro('redshift__some_macro', 'dbt_redshift') - # same name, different package - package_default_macro = mock_macro('default__some_macro', 'root') - package_rs_macro = mock_macro('redshift__some_macro', 'root') - manifest_fx.macros[dbt_macro.unique_id] = dbt_macro - manifest_fx.macros[rs_macro.unique_id] = rs_macro - manifest_fx.macros[package_default_macro.unique_id] = package_default_macro - manifest_fx.macros[package_rs_macro.unique_id] = package_rs_macro - return manifest_fx - @pytest.fixture -def redshift_adapter(config, get_adapter): - adapter = redshift.RedshiftAdapter(config) - inject_adapter(adapter, redshift.Plugin) - get_adapter.return_value = adapter - yield adapter - clear_plugin(redshift.Plugin) +def manifest_fx(config_postgres): + return mock_manifest(config_postgres) @pytest.fixture @@ -426,49 +376,49 @@ def postgres_adapter(config_postgres, get_adapter): clear_plugin(postgres.Plugin) -def test_query_header_context(config, manifest_fx): +def test_query_header_context(config_postgres, manifest_fx): ctx = manifest.generate_query_header_context( - config=config, + config=config_postgres, manifest=manifest_fx, ) assert_has_keys(REQUIRED_QUERY_HEADER_KEYS, MAYBE_KEYS, ctx) -def test_macro_runtime_context(config, manifest_fx, get_adapter, get_include_paths): +def test_macro_runtime_context(config_postgres, manifest_fx, get_adapter, get_include_paths): ctx = providers.generate_runtime_macro( macro=manifest_fx.macros['macro.root.macro_a'], - config=config, + config=config_postgres, manifest=manifest_fx, package_name='root', ) assert_has_keys(REQUIRED_MACRO_KEYS, MAYBE_KEYS, ctx) -def test_model_parse_context(config, manifest_fx, get_adapter, get_include_paths): +def test_model_parse_context(config_postgres, manifest_fx, get_adapter, get_include_paths): ctx = providers.generate_parser_model( model=mock_model(), - config=config, + config=config_postgres, manifest=manifest_fx, context_config=mock.MagicMock(), ) assert_has_keys(REQUIRED_MODEL_KEYS, MAYBE_KEYS, ctx) -def test_model_runtime_context(config, manifest_fx, get_adapter, get_include_paths): +def test_model_runtime_context(config_postgres, manifest_fx, get_adapter, get_include_paths): ctx = providers.generate_runtime_model( model=mock_model(), - config=config, + config=config_postgres, manifest=manifest_fx, ) assert_has_keys(REQUIRED_MODEL_KEYS, MAYBE_KEYS, ctx) -def test_docs_runtime_context(config): - ctx = docs.generate_runtime_docs(config, mock_model(), [], 'root') +def test_docs_runtime_context(config_postgres): + ctx = docs.generate_runtime_docs(config_postgres, mock_model(), [], 'root') assert_has_keys(REQUIRED_DOCS_KEYS, MAYBE_KEYS, ctx) -def test_macro_namespace_duplicates(config, manifest_fx): +def test_macro_namespace_duplicates(config_postgres, manifest_fx): mn = macros.MacroNamespaceBuilder( 'root', 'search', MacroStack(), ['dbt_postgres', 'dbt'] ) @@ -482,7 +432,7 @@ def test_macro_namespace_duplicates(config, manifest_fx): mn.add_macros(mock_macro('macro_a', 'dbt'), {}) -def test_macro_namespace(config, manifest_fx): +def test_macro_namespace(config_postgres, manifest_fx): mn = macros.MacroNamespaceBuilder( 'root', 'search', MacroStack(), ['dbt_postgres', 'dbt']) @@ -513,71 +463,3 @@ def test_macro_namespace(config, manifest_fx): assert result['dbt']['some_macro'].macro is pg_macro assert result['root']['some_macro'].macro is package_macro assert result['some_macro'].macro is package_macro - - -def test_resolve_specific(config, manifest_extended, redshift_adapter, get_include_paths): - rs_macro = manifest_extended.macros['macro.dbt_redshift.redshift__some_macro'] - package_rs_macro = manifest_extended.macros['macro.root.redshift__some_macro'] - - ctx = providers.generate_runtime_model( - model=mock_model(), - config=config, - manifest=manifest_extended, - ) - - # macro_a exists, but default__macro_a and redshift__macro_a do not - with pytest.raises(dbt.exceptions.CompilationException): - ctx['adapter'].dispatch('macro_a').macro - - assert ctx['adapter'].dispatch('some_macro').macro is package_rs_macro - assert ctx['adapter'].dispatch('some_macro', packages=[ - 'dbt']).macro is rs_macro - assert ctx['adapter'].dispatch('some_macro', packages=[ - 'root']).macro is package_rs_macro - assert ctx['adapter'].dispatch('some_macro', packages=[ - 'root', 'dbt']).macro is package_rs_macro - assert ctx['adapter'].dispatch('some_macro', packages=[ - 'dbt', 'root']).macro is rs_macro - - -def test_resolve_default(config_postgres, manifest_extended, postgres_adapter, get_include_paths): - dbt_macro = manifest_extended.macros['macro.dbt.default__some_macro'] - package_macro = manifest_extended.macros['macro.root.default__some_macro'] - - ctx = providers.generate_runtime_model( - model=mock_model(), - config=config_postgres, - manifest=manifest_extended, - ) - - # macro_a exists, but default__macro_a and redshift__macro_a do not - with pytest.raises(dbt.exceptions.CompilationException): - ctx['adapter'].dispatch('macro_a').macro - - assert ctx['adapter'].dispatch('some_macro').macro is package_macro - assert ctx['adapter'].dispatch('some_macro', packages=[ - 'dbt']).macro is dbt_macro - assert ctx['adapter'].dispatch('some_macro', packages=[ - 'root']).macro is package_macro - assert ctx['adapter'].dispatch('some_macro', packages=[ - 'root', 'dbt']).macro is package_macro - assert ctx['adapter'].dispatch('some_macro', packages=[ - 'dbt', 'root']).macro is dbt_macro - - -def test_resolve_errors(config, manifest_extended, redshift_adapter, get_include_paths): - ctx = providers.generate_runtime_model( - model=mock_model(), - config=config, - manifest=manifest_extended, - ) - with pytest.raises(dbt.exceptions.CompilationException) as exc: - ctx['adapter'].dispatch('no_default_exists') - assert 'default__no_default_exists' in str(exc.value) - assert 'redshift__no_default_exists' in str(exc.value) - - with pytest.raises(dbt.exceptions.CompilationException) as exc: - ctx['adapter'].dispatch('namespace.no_default_exists') - assert '"." is not a valid macro name component' in str(exc.value) - assert 'adapter.dispatch' in str(exc.value) - assert 'packages=["namespace"]' in str(exc.value) diff --git a/test/unit/test_macro_resolver.py b/test/unit/test_macro_resolver.py index e5fc03e365a..17e1aca6dca 100644 --- a/test/unit/test_macro_resolver.py +++ b/test/unit/test_macro_resolver.py @@ -1,25 +1,9 @@ -import itertools import unittest -import os -from typing import Set, Dict, Any from unittest import mock -import pytest - -# make sure 'redshift' is available -from dbt.adapters import postgres, redshift -from dbt.adapters import factory -from dbt.adapters.base import AdapterConfig from dbt.contracts.graph.parsed import ( - ParsedModelNode, NodeConfig, DependsOn, ParsedMacro + ParsedMacro ) -from dbt.context import base, target, configured, providers, docs, manifest, macros -from dbt.contracts.files import FileHash -from dbt.node_types import NodeType -import dbt.exceptions -from .utils import profile_from_dict, config_from_parts_or_dicts, inject_adapter, clear_plugin -from .mock_adapter import adapter_factory - from dbt.context.macro_resolver import MacroResolver @@ -34,6 +18,7 @@ def mock_macro(name, package_name): macro.name = name return macro + class TestMacroResolver(unittest.TestCase): def test_resolver(self): @@ -52,5 +37,3 @@ def test_resolver(self): resolver = MacroResolver(macros, 'my_test', ['one']) assert(resolver) self.assertEqual(resolver.get_macro_id('one', 'not_null'), 'macro.one.not_null') - - diff --git a/test/unit/test_parse_manifest.py b/test/unit/test_parse_manifest.py index ec804d1737e..3dd28623d58 100644 --- a/test/unit/test_parse_manifest.py +++ b/test/unit/test_parse_manifest.py @@ -33,7 +33,7 @@ def setUp(self): 'quoting': {}, 'outputs': { 'test': { - 'type': 'redshift', + 'type': 'postgres', 'host': 'localhost', 'schema': 'analytics', 'user': 'test', diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index e0bc28e7d6c..9f1672ff86c 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -69,7 +69,7 @@ def _generate_macros(self): def setUp(self): dbt.flags.WARN_ERROR = True - # HACK: this is needed since tracking events can + # HACK: this is needed since tracking events can # be sent when using the model parser tracking.do_not_track() @@ -80,7 +80,7 @@ def setUp(self): 'quoting': {}, 'outputs': { 'test': { - 'type': 'redshift', + 'type': 'postgres', 'host': 'localhost', 'schema': 'analytics', 'user': 'test', diff --git a/test/unit/test_redshift_adapter.py b/test/unit/test_redshift_adapter.py deleted file mode 100644 index 285c41ac1a6..00000000000 --- a/test/unit/test_redshift_adapter.py +++ /dev/null @@ -1,371 +0,0 @@ -import unittest -from unittest import mock -from unittest.mock import Mock - -import agate -import boto3 - -import dbt.adapters # noqa -import dbt.flags as flags - -from dbt.adapters.redshift import ( - RedshiftAdapter, - Plugin as RedshiftPlugin, -) -from dbt.clients import agate_helper -from dbt.exceptions import FailedToConnectException -from dbt.logger import GLOBAL_LOGGER as logger # noqa - -from .utils import config_from_parts_or_dicts, mock_connection, TestAdapterConversions, inject_adapter - - -@classmethod -def fetch_cluster_credentials(*args, **kwargs): - return { - 'DbUser': 'root', - 'DbPassword': 'tmp_password' - } - - -class TestRedshiftAdapter(unittest.TestCase): - - def setUp(self): - profile_cfg = { - 'outputs': { - 'test': { - 'type': 'redshift', - 'dbname': 'redshift', - 'user': 'root', - 'host': 'thishostshouldnotexist', - 'pass': 'password', - 'port': 5439, - 'schema': 'public' - } - }, - 'target': 'test' - } - - project_cfg = { - 'name': 'X', - 'version': '0.1', - 'profile': 'test', - 'project-root': '/tmp/dbt/does-not-exist', - 'quoting': { - 'identifier': False, - 'schema': True, - }, - 'config-version': 2, - } - - self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) - self._adapter = None - - @property - def adapter(self): - if self._adapter is None: - self._adapter = RedshiftAdapter(self.config) - inject_adapter(self._adapter, RedshiftPlugin) - return self._adapter - - def test_implicit_database_conn(self): - creds = RedshiftAdapter.ConnectionManager.get_credentials(self.config.credentials) - self.assertEqual(creds, self.config.credentials) - - def test_explicit_database_conn(self): - self.config.method = 'database' - - creds = RedshiftAdapter.ConnectionManager.get_credentials(self.config.credentials) - self.assertEqual(creds, self.config.credentials) - - def test_explicit_iam_conn(self): - self.config.credentials = self.config.credentials.replace( - method='iam', - cluster_id='my_redshift', - iam_duration_seconds=1200 - ) - - with mock.patch.object(RedshiftAdapter.ConnectionManager, 'fetch_cluster_credentials', new=fetch_cluster_credentials): - creds = RedshiftAdapter.ConnectionManager.get_credentials(self.config.credentials) - - expected_creds = self.config.credentials.replace(password='tmp_password') - self.assertEqual(creds, expected_creds) - - def test_iam_conn_optionals(self): - - profile_cfg = { - 'outputs': { - 'test': { - 'type': 'redshift', - 'dbname': 'redshift', - 'user': 'root', - 'host': 'thishostshouldnotexist', - 'port': 5439, - 'schema': 'public', - 'method': 'iam', - 'cluster_id': 'my_redshift', - 'db_groups': ["my_dbgroup"], - 'autocreate': True, - } - }, - 'target': 'test' - } - - config_from_parts_or_dicts(self.config, profile_cfg) - - def test_invalid_auth_method(self): - # we have to set method this way, otherwise it won't validate - self.config.credentials.method = 'badmethod' - - with self.assertRaises(FailedToConnectException) as context: - with mock.patch.object(RedshiftAdapter.ConnectionManager, 'fetch_cluster_credentials', new=fetch_cluster_credentials): - RedshiftAdapter.ConnectionManager.get_credentials(self.config.credentials) - - self.assertTrue('badmethod' in context.exception.msg) - - def test_invalid_iam_no_cluster_id(self): - self.config.credentials = self.config.credentials.replace(method='iam') - with self.assertRaises(FailedToConnectException) as context: - with mock.patch.object(RedshiftAdapter.ConnectionManager, 'fetch_cluster_credentials', new=fetch_cluster_credentials): - RedshiftAdapter.ConnectionManager.get_credentials(self.config.credentials) - - self.assertTrue("'cluster_id' must be provided" in context.exception.msg) - - def test_default_session_is_not_used_when_iam_used(self): - boto3.DEFAULT_SESSION = Mock() - self.config.credentials = self.config.credentials.replace(method='iam') - self.config.credentials.cluster_id = 'clusterid' - with mock.patch('dbt.adapters.redshift.connections.boto3.Session'): - RedshiftAdapter.ConnectionManager.get_credentials(self.config.credentials) - self.assertEqual(boto3.DEFAULT_SESSION.client.call_count, 0, - "The redshift client should not be created using the default session because the session object is not thread-safe") - - def test_default_session_is_not_used_when_iam_not_used(self): - boto3.DEFAULT_SESSION = Mock() - self.config.credentials = self.config.credentials.replace(method=None) - with mock.patch('dbt.adapters.redshift.connections.boto3.Session'): - RedshiftAdapter.ConnectionManager.get_credentials(self.config.credentials) - self.assertEqual(boto3.DEFAULT_SESSION.client.call_count, 0, - "The redshift client should not be created using the default session because the session object is not thread-safe") - - def test_cancel_open_connections_empty(self): - self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) - - def test_cancel_open_connections_master(self): - key = self.adapter.connections.get_thread_identifier() - self.adapter.connections.thread_connections[key] = mock_connection('master') - self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) - - def test_cancel_open_connections_single(self): - master = mock_connection('master') - model = mock_connection('model') - model.handle.get_backend_pid.return_value = 42 - - key = self.adapter.connections.get_thread_identifier() - self.adapter.connections.thread_connections.update({ - key: master, - 1: model, - }) - with mock.patch.object(self.adapter.connections, 'add_query') as add_query: - query_result = mock.MagicMock() - add_query.return_value = (None, query_result) - - self.assertEqual(len(list(self.adapter.cancel_open_connections())), 1) - - add_query.assert_called_once_with('select pg_terminate_backend(42)') - - master.handle.get_backend_pid.assert_not_called() - - @mock.patch('dbt.adapters.postgres.connections.psycopg2') - def test_default_keepalive(self, psycopg2): - connection = self.adapter.acquire_connection('dummy') - - psycopg2.connect.assert_not_called() - connection.handle - psycopg2.connect.assert_called_once_with( - dbname='redshift', - user='root', - host='thishostshouldnotexist', - password='password', - port=5439, - connect_timeout=10, - keepalives_idle=240, - application_name='dbt' - ) - - @mock.patch('dbt.adapters.postgres.connections.psycopg2') - def test_changed_keepalive(self, psycopg2): - self.config.credentials = self.config.credentials.replace(keepalives_idle=256) - connection = self.adapter.acquire_connection('dummy') - - psycopg2.connect.assert_not_called() - connection.handle - psycopg2.connect.assert_called_once_with( - dbname='redshift', - user='root', - host='thishostshouldnotexist', - password='password', - port=5439, - connect_timeout=10, - keepalives_idle=256, - application_name='dbt') - - @mock.patch('dbt.adapters.postgres.connections.psycopg2') - def test_search_path(self, psycopg2): - self.config.credentials = self.config.credentials.replace(search_path="test") - connection = self.adapter.acquire_connection('dummy') - - psycopg2.connect.assert_not_called() - connection.handle - psycopg2.connect.assert_called_once_with( - dbname='redshift', - user='root', - host='thishostshouldnotexist', - password='password', - port=5439, - connect_timeout=10, - options="-c search_path=test", - keepalives_idle=240, - application_name='dbt') - - @mock.patch('dbt.adapters.postgres.connections.psycopg2') - def test_search_path_with_space(self, psycopg2): - self.config.credentials = self.config.credentials.replace(search_path="test test") - connection = self.adapter.acquire_connection('dummy') - - psycopg2.connect.assert_not_called() - connection.handle - psycopg2.connect.assert_called_once_with( - dbname='redshift', - user='root', - host='thishostshouldnotexist', - password='password', - port=5439, - connect_timeout=10, - options=r"-c search_path=test\ test", - keepalives_idle=240, - application_name='dbt') - - @mock.patch('dbt.adapters.postgres.connections.psycopg2') - def test_set_zero_keepalive(self, psycopg2): - self.config.credentials = self.config.credentials.replace(keepalives_idle=0) - connection = self.adapter.acquire_connection('dummy') - - psycopg2.connect.assert_not_called() - connection.handle - psycopg2.connect.assert_called_once_with( - dbname='redshift', - user='root', - host='thishostshouldnotexist', - password='password', - port=5439, - connect_timeout=10, - application_name='dbt') - - def test_dbname_verification_is_case_insensitive(self): - # Override adapter settings from setUp() - profile_cfg = { - 'outputs': { - 'test': { - 'type': 'redshift', - 'dbname': 'Redshift', - 'user': 'root', - 'host': 'thishostshouldnotexist', - 'pass': 'password', - 'port': 5439, - 'schema': 'public' - } - }, - 'target': 'test' - } - - project_cfg = { - 'name': 'X', - 'version': '0.1', - 'profile': 'test', - 'project-root': '/tmp/dbt/does-not-exist', - 'quoting': { - 'identifier': False, - 'schema': True, - }, - 'config-version': 2, - } - self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) - self.adapter.cleanup_connections() - self._adapter = RedshiftAdapter(self.config) - self.adapter.verify_database('redshift') - - -class TestRedshiftAdapterConversions(TestAdapterConversions): - def test_convert_text_type(self): - rows = [ - ['', 'a1', 'stringval1'], - ['', 'a2', 'stringvalasdfasdfasdfa'], - ['', 'a3', 'stringval3'], - ] - agate_table = self._make_table_of(rows, agate.Text) - expected = ['varchar(64)', 'varchar(2)', 'varchar(22)'] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_text_type(agate_table, col_idx) == expect - - def test_convert_number_type(self): - rows = [ - ['', '23.98', '-1'], - ['', '12.78', '-2'], - ['', '79.41', '-3'], - ] - agate_table = self._make_table_of(rows, agate.Number) - expected = ['integer', 'float8', 'integer'] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_number_type(agate_table, col_idx) == expect - - def test_convert_boolean_type(self): - rows = [ - ['', 'false', 'true'], - ['', 'false', 'false'], - ['', 'false', 'true'], - ] - agate_table = self._make_table_of(rows, agate.Boolean) - expected = ['boolean', 'boolean', 'boolean'] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_boolean_type(agate_table, col_idx) == expect - - def test_convert_datetime_type(self): - rows = [ - ['', '20190101T01:01:01Z', '2019-01-01 01:01:01'], - ['', '20190102T01:01:01Z', '2019-01-01 01:01:01'], - ['', '20190103T01:01:01Z', '2019-01-01 01:01:01'], - ] - agate_table = self._make_table_of(rows, [agate.DateTime, agate_helper.ISODateTime, agate.DateTime]) - expected = ['timestamp without time zone', 'timestamp without time zone', 'timestamp without time zone'] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_datetime_type(agate_table, col_idx) == expect - - def test_convert_date_type(self): - rows = [ - ['', '2019-01-01', '2019-01-04'], - ['', '2019-01-02', '2019-01-04'], - ['', '2019-01-03', '2019-01-04'], - ] - agate_table = self._make_table_of(rows, agate.Date) - expected = ['date', 'date', 'date'] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_date_type(agate_table, col_idx) == expect - - def test_convert_time_type(self): - # dbt's default type testers actually don't have a TimeDelta at all. - agate.TimeDelta - rows = [ - ['', '120s', '10s'], - ['', '3m', '11s'], - ['', '1h', '12s'], - ] - agate_table = self._make_table_of(rows, agate.TimeDelta) - expected = ['varchar(24)', 'varchar(24)', 'varchar(24)'] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_time_type(agate_table, col_idx) == expect - - -# convert_boolean_type -# convert_datetime_type -# convert_date_type -# convert_time_type diff --git a/test/unit/test_snowflake_adapter.py b/test/unit/test_snowflake_adapter.py deleted file mode 100644 index 472a229d315..00000000000 --- a/test/unit/test_snowflake_adapter.py +++ /dev/null @@ -1,583 +0,0 @@ -import agate -import re -import unittest -from contextlib import contextmanager -from unittest import mock - -import dbt.flags as flags - -from dbt.adapters.snowflake import SnowflakeAdapter -from dbt.adapters.snowflake import Plugin as SnowflakePlugin -from dbt.adapters.snowflake.column import SnowflakeColumn -from dbt.adapters.base.query_headers import MacroQueryStringSetter -from dbt.contracts.files import FileHash -from dbt.contracts.graph.manifest import ManifestStateCheck -from dbt.clients import agate_helper -from dbt.logger import GLOBAL_LOGGER as logger # noqa -from snowflake import connector as snowflake_connector - -from .utils import config_from_parts_or_dicts, inject_adapter, mock_connection, TestAdapterConversions, load_internal_manifest_macros - - -class TestSnowflakeAdapter(unittest.TestCase): - def setUp(self): - profile_cfg = { - 'outputs': { - 'test': { - 'type': 'snowflake', - 'account': 'test_account', - 'user': 'test_user', - 'database': 'test_database', - 'warehouse': 'test_warehouse', - 'schema': 'public', - }, - }, - 'target': 'test', - } - - project_cfg = { - 'name': 'X', - 'version': '0.1', - 'profile': 'test', - 'project-root': '/tmp/dbt/does-not-exist', - 'quoting': { - 'identifier': False, - 'schema': True, - }, - 'query-comment': 'dbt', - 'config-version': 2, - } - self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) - self.assertEqual(self.config.query_comment.comment, 'dbt') - self.assertEqual(self.config.query_comment.append, False) - - self.handle = mock.MagicMock( - spec=snowflake_connector.SnowflakeConnection) - self.cursor = self.handle.cursor.return_value - self.mock_execute = self.cursor.execute - self.patcher = mock.patch( - 'dbt.adapters.snowflake.connections.snowflake.connector.connect' - ) - self.snowflake = self.patcher.start() - - # Create the Manifest.state_check patcher - @mock.patch('dbt.parser.manifest.ManifestLoader.build_manifest_state_check') - def _mock_state_check(self): - config = self.root_project - all_projects = self.all_projects - return ManifestStateCheck( - vars_hash=FileHash.from_contents('vars'), - project_hashes={name: FileHash.from_contents(name) for name in all_projects}, - profile_hash=FileHash.from_contents('profile'), - ) - self.load_state_check = mock.patch('dbt.parser.manifest.ManifestLoader.build_manifest_state_check') - self.mock_state_check = self.load_state_check.start() - self.mock_state_check.side_effect = _mock_state_check - - self.snowflake.return_value = self.handle - self.adapter = SnowflakeAdapter(self.config) - self.adapter._macro_manifest_lazy = load_internal_manifest_macros(self.config) - self.adapter.connections.query_header = MacroQueryStringSetter(self.config, self.adapter._macro_manifest_lazy) - - self.qh_patch = mock.patch.object(self.adapter.connections.query_header, 'add') - self.mock_query_header_add = self.qh_patch.start() - self.mock_query_header_add.side_effect = lambda q: '/* dbt */\n{}'.format(q) - - self.adapter.acquire_connection() - inject_adapter(self.adapter, SnowflakePlugin) - - def tearDown(self): - # we want a unique self.handle every time. - self.adapter.cleanup_connections() - self.qh_patch.stop() - self.patcher.stop() - self.load_state_check.stop() - - def test_quoting_on_drop_schema(self): - relation = SnowflakeAdapter.Relation.create( - database='test_database', - schema='test_schema', - quote_policy=self.adapter.config.quoting - ) - self.adapter.drop_schema(relation) - - self.mock_execute.assert_has_calls([ - mock.call('/* dbt */\ndrop schema if exists test_database."test_schema" cascade', None) - ]) - - def test_quoting_on_drop(self): - relation = self.adapter.Relation.create( - database='test_database', - schema='test_schema', - identifier='test_table', - type='table', - quote_policy=self.adapter.config.quoting, - ) - self.adapter.drop_relation(relation) - - self.mock_execute.assert_has_calls([ - mock.call( - '/* dbt */\ndrop table if exists test_database."test_schema".test_table cascade', - None - ) - ]) - - def test_quoting_on_truncate(self): - relation = self.adapter.Relation.create( - database='test_database', - schema='test_schema', - identifier='test_table', - type='table', - quote_policy=self.adapter.config.quoting, - ) - self.adapter.truncate_relation(relation) - - # no query comment because wrapped in begin; + commit; for explicit DML - self.mock_execute.assert_has_calls([ - mock.call('/* dbt */\nbegin;', None), - mock.call('truncate table test_database."test_schema".test_table\n ;', None), - mock.call('commit;', None) - ]) - - def test_quoting_on_rename(self): - from_relation = self.adapter.Relation.create( - database='test_database', - schema='test_schema', - identifier='table_a', - type='table', - quote_policy=self.adapter.config.quoting, - ) - to_relation = self.adapter.Relation.create( - database='test_database', - schema='test_schema', - identifier='table_b', - type='table', - quote_policy=self.adapter.config.quoting, - ) - - self.adapter.rename_relation( - from_relation=from_relation, - to_relation=to_relation - ) - self.mock_execute.assert_has_calls([ - mock.call( - '/* dbt */\nalter table test_database."test_schema".table_a rename to test_database."test_schema".table_b', - None - ) - ]) - - @contextmanager - def current_warehouse(self, response): - # there is probably some elegant way built into mock.patch to do this - fetchall_return = self.cursor.fetchall.return_value - execute_side_effect = self.mock_execute.side_effect - - def execute_effect(sql, *args, **kwargs): - if sql == '/* dbt */\nselect current_warehouse() as warehouse': - self.cursor.description = [['name']] - self.cursor.fetchall.return_value = [[response]] - else: - self.cursor.description = None - self.cursor.fetchall.return_value = fetchall_return - return self.mock_execute.return_value - - self.mock_execute.side_effect = execute_effect - try: - yield - finally: - self.cursor.fetchall.return_value = fetchall_return - self.mock_execute.side_effect = execute_side_effect - - def _strip_transactions(self): - result = [] - for call_args in self.mock_execute.call_args_list: - args, kwargs = tuple(call_args) - is_transactional = ( - len(kwargs) == 0 and - len(args) == 2 and - args[1] is None and - args[0] in {'BEGIN', 'COMMIT'} - ) - if not is_transactional: - result.append(call_args) - return result - - def test_pre_post_hooks_warehouse(self): - with self.current_warehouse('warehouse'): - config = {'snowflake_warehouse': 'other_warehouse'} - result = self.adapter.pre_model_hook(config) - self.assertIsNotNone(result) - calls = [ - mock.call('/* dbt */\nselect current_warehouse() as warehouse', None), - mock.call('/* dbt */\nuse warehouse other_warehouse', None) - ] - self.mock_execute.assert_has_calls(calls) - self.adapter.post_model_hook(config, result) - calls.append(mock.call('/* dbt */\nuse warehouse warehouse', None)) - self.mock_execute.assert_has_calls(calls) - - def test_pre_post_hooks_no_warehouse(self): - with self.current_warehouse('warehouse'): - config = {} - result = self.adapter.pre_model_hook(config) - self.assertIsNone(result) - self.mock_execute.assert_not_called() - self.adapter.post_model_hook(config, result) - self.mock_execute.assert_not_called() - - def test_cancel_open_connections_empty(self): - self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) - - def test_cancel_open_connections_master(self): - key = self.adapter.connections.get_thread_identifier() - self.adapter.connections.thread_connections[key] = mock_connection('master') - self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) - - def test_cancel_open_connections_single(self): - master = mock_connection('master') - model = mock_connection('model') - model.handle.session_id = 42 - - key = self.adapter.connections.get_thread_identifier() - self.adapter.connections.thread_connections.update({ - key: master, - 1: model, - }) - with mock.patch.object(self.adapter.connections, 'add_query') as add_query: - query_result = mock.MagicMock() - add_query.return_value = (None, query_result) - - self.assertEqual( - len(list(self.adapter.cancel_open_connections())), 1) - - add_query.assert_called_once_with('select system$abort_session(42)') - - def test_client_session_keep_alive_false_by_default(self): - conn = self.adapter.connections.set_connection_name(name='new_connection_with_new_config') - - self.snowflake.assert_not_called() - conn.handle - self.snowflake.assert_has_calls([ - mock.call( - account='test_account', autocommit=True, - client_session_keep_alive=False, database='test_database', - role=None, schema='public', user='test_user', - warehouse='test_warehouse', private_key=None, application='dbt') - ]) - - def test_client_session_keep_alive_true(self): - self.config.credentials = self.config.credentials.replace( - client_session_keep_alive=True) - self.adapter = SnowflakeAdapter(self.config) - conn = self.adapter.connections.set_connection_name(name='new_connection_with_new_config') - - self.snowflake.assert_not_called() - conn.handle - self.snowflake.assert_has_calls([ - mock.call( - account='test_account', autocommit=True, - client_session_keep_alive=True, database='test_database', - role=None, schema='public', user='test_user', - warehouse='test_warehouse', private_key=None, application='dbt') - ]) - - def test_user_pass_authentication(self): - self.config.credentials = self.config.credentials.replace( - password='test_password', - ) - self.adapter = SnowflakeAdapter(self.config) - conn = self.adapter.connections.set_connection_name(name='new_connection_with_new_config') - - self.snowflake.assert_not_called() - conn.handle - self.snowflake.assert_has_calls([ - mock.call( - account='test_account', autocommit=True, - client_session_keep_alive=False, database='test_database', - password='test_password', role=None, schema='public', - user='test_user', warehouse='test_warehouse', private_key=None, - application='dbt') - ]) - - def test_authenticator_user_pass_authentication(self): - self.config.credentials = self.config.credentials.replace( - password='test_password', - authenticator='test_sso_url', - ) - self.adapter = SnowflakeAdapter(self.config) - conn = self.adapter.connections.set_connection_name(name='new_connection_with_new_config') - - self.snowflake.assert_not_called() - conn.handle - self.snowflake.assert_has_calls([ - mock.call( - account='test_account', autocommit=True, - client_session_keep_alive=False, database='test_database', - password='test_password', role=None, schema='public', - user='test_user', warehouse='test_warehouse', - authenticator='test_sso_url', private_key=None, - application='dbt', client_store_temporary_credential=True) - ]) - - def test_authenticator_externalbrowser_authentication(self): - self.config.credentials = self.config.credentials.replace( - authenticator='externalbrowser' - ) - self.adapter = SnowflakeAdapter(self.config) - conn = self.adapter.connections.set_connection_name(name='new_connection_with_new_config') - - self.snowflake.assert_not_called() - conn.handle - self.snowflake.assert_has_calls([ - mock.call( - account='test_account', autocommit=True, - client_session_keep_alive=False, database='test_database', - role=None, schema='public', user='test_user', - warehouse='test_warehouse', authenticator='externalbrowser', - private_key=None, application='dbt', client_store_temporary_credential=True) - ]) - - def test_authenticator_oauth_authentication(self): - self.config.credentials = self.config.credentials.replace( - authenticator='oauth', - token='my-oauth-token', - ) - self.adapter = SnowflakeAdapter(self.config) - conn = self.adapter.connections.set_connection_name(name='new_connection_with_new_config') - - self.snowflake.assert_not_called() - conn.handle - self.snowflake.assert_has_calls([ - mock.call( - account='test_account', autocommit=True, - client_session_keep_alive=False, database='test_database', - role=None, schema='public', user='test_user', - warehouse='test_warehouse', authenticator='oauth', token='my-oauth-token', - private_key=None, application='dbt', client_store_temporary_credential=True) - ]) - - @mock.patch('dbt.adapters.snowflake.SnowflakeCredentials._get_private_key', return_value='test_key') - def test_authenticator_private_key_authentication(self, mock_get_private_key): - self.config.credentials = self.config.credentials.replace( - private_key_path='/tmp/test_key.p8', - private_key_passphrase='p@ssphr@se', - ) - - self.adapter = SnowflakeAdapter(self.config) - conn = self.adapter.connections.set_connection_name(name='new_connection_with_new_config') - - self.snowflake.assert_not_called() - conn.handle - self.snowflake.assert_has_calls([ - mock.call( - account='test_account', autocommit=True, - client_session_keep_alive=False, database='test_database', - role=None, schema='public', user='test_user', - warehouse='test_warehouse', private_key='test_key', - application='dbt') - ]) - - @mock.patch('dbt.adapters.snowflake.SnowflakeCredentials._get_private_key', return_value='test_key') - def test_authenticator_private_key_authentication_no_passphrase(self, mock_get_private_key): - self.config.credentials = self.config.credentials.replace( - private_key_path='/tmp/test_key.p8', - private_key_passphrase=None, - ) - - self.adapter = SnowflakeAdapter(self.config) - conn = self.adapter.connections.set_connection_name(name='new_connection_with_new_config') - - self.snowflake.assert_not_called() - conn.handle - self.snowflake.assert_has_calls([ - mock.call( - account='test_account', autocommit=True, - client_session_keep_alive=False, database='test_database', - role=None, schema='public', user='test_user', - warehouse='test_warehouse', private_key='test_key', - application='dbt') - ]) - - -class TestSnowflakeAdapterConversions(TestAdapterConversions): - def test_convert_text_type(self): - rows = [ - ['', 'a1', 'stringval1'], - ['', 'a2', 'stringvalasdfasdfasdfa'], - ['', 'a3', 'stringval3'], - ] - agate_table = self._make_table_of(rows, agate.Text) - expected = ['text', 'text', 'text'] - for col_idx, expect in enumerate(expected): - assert SnowflakeAdapter.convert_text_type(agate_table, col_idx) == expect - - def test_convert_number_type(self): - rows = [ - ['', '23.98', '-1'], - ['', '12.78', '-2'], - ['', '79.41', '-3'], - ] - agate_table = self._make_table_of(rows, agate.Number) - expected = ['integer', 'float8', 'integer'] - for col_idx, expect in enumerate(expected): - assert SnowflakeAdapter.convert_number_type(agate_table, col_idx) == expect - - def test_convert_boolean_type(self): - rows = [ - ['', 'false', 'true'], - ['', 'false', 'false'], - ['', 'false', 'true'], - ] - agate_table = self._make_table_of(rows, agate.Boolean) - expected = ['boolean', 'boolean', 'boolean'] - for col_idx, expect in enumerate(expected): - assert SnowflakeAdapter.convert_boolean_type(agate_table, col_idx) == expect - - def test_convert_datetime_type(self): - rows = [ - ['', '20190101T01:01:01Z', '2019-01-01 01:01:01'], - ['', '20190102T01:01:01Z', '2019-01-01 01:01:01'], - ['', '20190103T01:01:01Z', '2019-01-01 01:01:01'], - ] - agate_table = self._make_table_of(rows, [agate.DateTime, agate_helper.ISODateTime, agate.DateTime]) - expected = ['timestamp without time zone', 'timestamp without time zone', 'timestamp without time zone'] - for col_idx, expect in enumerate(expected): - assert SnowflakeAdapter.convert_datetime_type(agate_table, col_idx) == expect - - def test_convert_date_type(self): - rows = [ - ['', '2019-01-01', '2019-01-04'], - ['', '2019-01-02', '2019-01-04'], - ['', '2019-01-03', '2019-01-04'], - ] - agate_table = self._make_table_of(rows, agate.Date) - expected = ['date', 'date', 'date'] - for col_idx, expect in enumerate(expected): - assert SnowflakeAdapter.convert_date_type(agate_table, col_idx) == expect - - def test_convert_time_type(self): - # dbt's default type testers actually don't have a TimeDelta at all. - agate.TimeDelta - rows = [ - ['', '120s', '10s'], - ['', '3m', '11s'], - ['', '1h', '12s'], - ] - agate_table = self._make_table_of(rows, agate.TimeDelta) - expected = ['time', 'time', 'time'] - for col_idx, expect in enumerate(expected): - assert SnowflakeAdapter.convert_time_type(agate_table, col_idx) == expect - - -class TestSnowflakeColumn(unittest.TestCase): - def test_text_from_description(self): - col = SnowflakeColumn.from_description('my_col', 'TEXT') - assert col.column == 'my_col' - assert col.dtype == 'TEXT' - assert col.char_size is None - assert col.numeric_precision is None - assert col.numeric_scale is None - assert col.is_float() is False - assert col.is_number() is False - assert col.is_numeric() is False - assert col.is_string() is True - assert col.is_integer() is False - assert col.string_size() == 16777216 - - col = SnowflakeColumn.from_description('my_col', 'VARCHAR') - assert col.column == 'my_col' - assert col.dtype == 'VARCHAR' - assert col.char_size is None - assert col.numeric_precision is None - assert col.numeric_scale is None - assert col.is_float() is False - assert col.is_number() is False - assert col.is_numeric() is False - assert col.is_string() is True - assert col.is_integer() is False - assert col.string_size() == 16777216 - - def test_sized_varchar_from_description(self): - col = SnowflakeColumn.from_description('my_col', 'VARCHAR(256)') - assert col.column == 'my_col' - assert col.dtype == 'VARCHAR' - assert col.char_size == 256 - assert col.numeric_precision is None - assert col.numeric_scale is None - assert col.is_float() is False - assert col.is_number() is False - assert col.is_numeric() is False - assert col.is_string() is True - assert col.is_integer() is False - assert col.string_size() == 256 - - def test_sized_decimal_from_description(self): - col = SnowflakeColumn.from_description('my_col', 'DECIMAL(1, 0)') - assert col.column == 'my_col' - assert col.dtype == 'DECIMAL' - assert col.char_size is None - assert col.numeric_precision == 1 - assert col.numeric_scale == 0 - assert col.is_float() is False - assert col.is_number() is True - assert col.is_numeric() is True - assert col.is_string() is False - assert col.is_integer() is False - - def test_float_from_description(self): - col = SnowflakeColumn.from_description('my_col', 'FLOAT8') - assert col.column == 'my_col' - assert col.dtype == 'FLOAT8' - assert col.char_size is None - assert col.numeric_precision is None - assert col.numeric_scale is None - assert col.is_float() is True - assert col.is_number() is True - assert col.is_numeric() is False - assert col.is_string() is False - assert col.is_integer() is False - - -class SnowflakeConnectionsTest(unittest.TestCase): - def test_comment_stripping_regex(self): - pattern = r'(\".*?\"|\'.*?\')|(/\*.*?\*/|--[^\r\n]*$)' - comment1 = '-- just comment' - comment2 = '/* just comment */' - query1 = 'select 1; -- comment' - query2 = 'select 1; /* comment */' - query3 = 'select 1; -- comment\nselect 2; /* comment */ ' - query4 = 'select \n1; -- comment\nselect \n2; /* comment */ ' - query5 = 'select 1; -- comment \nselect 2; -- comment \nselect 3; -- comment' - - stripped_comment1 = re.sub(re.compile(pattern, re.MULTILINE), - '', comment1).strip() - - stripped_comment2 = re.sub(re.compile(pattern, re.MULTILINE), - '', comment2).strip() - - stripped_query1 = re.sub(re.compile(pattern, re.MULTILINE), - '', query1).strip() - - stripped_query2 = re.sub(re.compile(pattern, re.MULTILINE), - '', query2).strip() - - stripped_query3 = re.sub(re.compile(pattern, re.MULTILINE), - '', query3).strip() - - stripped_query4 = re.sub(re.compile(pattern, re.MULTILINE), - '', query4).strip() - - stripped_query5 = re.sub(re.compile(pattern, re.MULTILINE), - '', query5).strip() - - expected_query_3 = 'select 1; \nselect 2;' - expected_query_4 = 'select \n1; \nselect \n2;' - expected_query_5 = 'select 1; \nselect 2; \nselect 3;' - - self.assertEqual('', stripped_comment1) - self.assertEqual('', stripped_comment2) - self.assertEqual('select 1;', stripped_query1) - self.assertEqual('select 1;', stripped_query2) - self.assertEqual(expected_query_3, stripped_query3) - self.assertEqual(expected_query_4, stripped_query4) - self.assertEqual(expected_query_5, stripped_query5) diff --git a/tox.ini b/tox.ini index bce3be1ea47..3f5a93eb200 100644 --- a/tox.ini +++ b/tox.ini @@ -8,10 +8,7 @@ basepython = python3.8 skip_install = true commands = flake8 --select=E,W,F --ignore=W504,E741 --max-line-length 99 \ core/dbt \ - plugins/bigquery/dbt \ - plugins/snowflake/dbt \ - plugins/postgres/dbt \ - plugins/redshift/dbt + plugins/postgres/dbt deps = -rdev-requirements.txt @@ -33,25 +30,17 @@ deps = -rdev-requirements.txt -reditable-requirements.txt -[testenv:{integration,py36,py37,py38,py39,py}-{postgres,redshift,snowflake,bigquery}] +[testenv:{integration,py36,py37,py38,py39,py}-{postgres}] description = adapter plugin integration testing skip_install = true -passenv = DBT_* REDSHIFT_TEST_* BIGQUERY_TEST_* SNOWFLAKE_TEST_* POSTGRES_TEST_* PYTEST_ADDOPTS +passenv = DBT_* POSTGRES_TEST_* PYTEST_ADDOPTS commands = postgres: {envpython} -m pytest {posargs} -m profile_postgres test/integration postgres: {envpython} -m pytest {posargs} --profile=postgres test/rpc - redshift: {envpython} -m pytest {posargs} -m profile_redshift test/integration - snowflake: {envpython} -m pytest {posargs} -m profile_snowflake test/integration - snowflake: {envpython} -m pytest {posargs} --profile=snowflake test/rpc - bigquery: {envpython} -m pytest {posargs} -m profile_bigquery test/integration deps = -rdev-requirements.txt -e./core postgres: -e./plugins/postgres - redshift: -e./plugins/redshift - redshift: -e./plugins/postgres - snowflake: -e./plugins/snowflake - bigquery: -e./plugins/bigquery [pytest] env_files =