From 4dd16bf786b4c79c04409835fc96ac0e717eb3b7 Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Sat, 11 Jan 2025 14:57:55 +0100 Subject: [PATCH 01/12] Add project_by_project_table materialization --- .../unreleased/Features-20250111-142613.yaml | 6 ++ .github/workflows/pr_lint_models.yml | 2 +- .github/workflows/pr_run_models.yml | 2 +- macros/project_by_project_table.sql | 34 +++++++ .../information_schema_object_privileges.sql | 29 +----- .../information_schema_bi_capacities.sql | 27 +----- ...information_schema_bi_capacity_changes.sql | 31 +------ ...ation_schema_effective_project_options.sql | 20 +---- ...nformation_schema_organization_options.sql | 18 +--- ...on_schema_organization_options_changes.sql | 20 +---- .../information_schema_project_options.sql | 18 +--- ...rmation_schema_project_options_changes.sql | 20 +---- .../datasets/information_schema_schemata.sql | 33 +------ .../information_schema_schemata_links.sql | 31 +------ .../information_schema_schemata_options.sql | 27 +----- .../information_schema_schemata_replicas.sql | 37 +------- ...emata_replicas_by_failover_reservation.sql | 34 +------ ...nformation_schema_shared_dataset_usage.sql | 49 +---------- .../google/jobs/information_schema_jobs.sql | 88 +------------------ .../information_schema_jobs_by_folder.sql | 83 +---------------- ...nformation_schema_jobs_by_organization.sql | 83 +---------------- .../information_schema_jobs_by_project.sql | 83 +---------------- .../jobs/information_schema_jobs_by_user.sql | 85 +----------------- .../information_schema_jobs_timeline.sql | 55 +----------- .../information_schema_jobs_timeline.yml | 22 ++++- ...rmation_schema_jobs_timeline_by_folder.sql | 57 +----------- ...rmation_schema_jobs_timeline_by_folder.yml | 22 ++++- ...n_schema_jobs_timeline_by_organization.sql | 57 +----------- ...n_schema_jobs_timeline_by_organization.yml | 19 +++- ...formation_schema_jobs_timeline_by_user.sql | 55 +----------- ...formation_schema_jobs_timeline_by_user.yml | 22 ++++- .../information_schema_insights.sql | 43 +-------- .../information_schema_recommendations.sql | 43 +-------- ...schema_recommendations_by_organization.sql | 43 +-------- .../information_schema_assignment_changes.sql | 41 +-------- .../information_schema_assignments.sql | 35 +------- ...ion_schema_capacity_commitment_changes.sql | 47 +--------- ...nformation_schema_capacity_commitments.sql | 37 +------- ...information_schema_reservation_changes.sql | 39 +------- .../information_schema_reservations.sql | 35 +------- ...formation_schema_reservations_timeline.sql | 35 +------- ...formation_schema_reservations_timeline.yml | 36 ++++---- .../information_schema_parameters.sql | 37 +------- .../information_schema_routine_options.sql | 29 +----- .../routines/information_schema_routines.sql | 51 +---------- ...nformation_schema_search_index_columns.sql | 29 +----- .../information_schema_search_indexes.sql | 49 +---------- ...information_schema_sessions_by_project.sql | 33 +------ .../information_schema_sessions_by_user.sql | 33 +------ .../information_schema_streaming_timeline.sql | 35 +------- ...on_schema_streaming_timeline_by_folder.sql | 35 +------- ...ema_streaming_timeline_by_organization.sql | 35 +------- .../information_schema_column_field_paths.sql | 35 +------- .../tables/information_schema_columns.sql | 53 +---------- ...rmation_schema_constraint_column_usage.sql | 35 ++------ .../information_schema_key_column_usage.sql | 39 ++------ .../tables/information_schema_partitions.sql | 37 ++------ .../information_schema_table_constraints.sql | 37 +------- .../information_schema_table_options.sql | 29 +----- .../information_schema_table_snapshots.sql | 31 +------ .../information_schema_table_storage.sql | 59 +------------ ...n_schema_table_storage_by_organization.sql | 59 +------------ ...on_schema_table_storage_usage_timeline.sql | 39 +------- ...storage_usage_timeline_by_organization.sql | 39 +------- .../tables/information_schema_tables.sql | 57 +----------- ...nformation_schema_vector_index_columns.sql | 29 +----- ...nformation_schema_vector_index_options.sql | 31 +------ .../information_schema_vector_indexes.sql | 47 +--------- .../information_schema_materialized_views.sql | 29 +----- .../google/views/information_schema_views.sql | 29 +----- .../information_schema_write_api_timeline.sql | 37 +------- ...on_schema_write_api_timeline_by_folder.sql | 37 +------- ...ema_write_api_timeline_by_organization.sql | 37 +------- 73 files changed, 279 insertions(+), 2515 deletions(-) create mode 100644 .changes/unreleased/Features-20250111-142613.yaml create mode 100644 macros/project_by_project_table.sql diff --git a/.changes/unreleased/Features-20250111-142613.yaml b/.changes/unreleased/Features-20250111-142613.yaml new file mode 100644 index 0000000..d3c2790 --- /dev/null +++ b/.changes/unreleased/Features-20250111-142613.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Improve the scalability of the project approach by factoring them into a table +time: 2025-01-11T14:26:13.652366+01:00 +custom: + Author: Kayrnt + Issue: "" diff --git a/.github/workflows/pr_lint_models.yml b/.github/workflows/pr_lint_models.yml index 719a02b..44263e6 100644 --- a/.github/workflows/pr_lint_models.yml +++ b/.github/workflows/pr_lint_models.yml @@ -41,7 +41,7 @@ jobs: python-version: '3.11' - name: Install Python packages - run: python -m pip install dbt-bigquery~=1.8.2 sqlfluff-templater-dbt + run: python -m pip install dbt-bigquery~=1.9.1 sqlfluff-templater-dbt - name: Write keyfile if secret is defined run: | diff --git a/.github/workflows/pr_run_models.yml b/.github/workflows/pr_run_models.yml index e1cc408..aed5f1b 100644 --- a/.github/workflows/pr_run_models.yml +++ b/.github/workflows/pr_run_models.yml @@ -41,7 +41,7 @@ jobs: python-version: '3.11' - name: Install Python packages - run: python -m pip install dbt-bigquery~=1.8.2 sqlfluff-templater-dbt + run: python -m pip install dbt-bigquery~=1.9.1 sqlfluff-templater-dbt - name: Write keyfile if secret is defined run: | diff --git a/macros/project_by_project_table.sql b/macros/project_by_project_table.sql new file mode 100644 index 0000000..eaf1115 --- /dev/null +++ b/macros/project_by_project_table.sql @@ -0,0 +1,34 @@ +{% macro project_by_project_table() %} + {%- materialization 'project_by_project_table', adapter='bigquery' -%} + + {% set target_relation = this %} + {% set existing_relation = load_relation(this) %} + {% set tmp_relation = make_temp_relation(this) %} + {% set projects = project_list() %} + + {{ run_hooks(pre_hooks) }} + + -- Create the table if it doesn't exist + {% if existing_relation is none %} + {% set build_sql = create_table_as(False, target_relation, sql) %} + {% do run_query(build_sql) %} + {% endif %} + + -- If we have projects, process them one by one + {% if projects|length > 0 %} + {% for project in projects %} + {% set project_sql = sql | replace('`region-', '`' ~ project | trim ~ '`.`region-') %} + {% set insert_sql %} + INSERT INTO {{ target_relation }} + {{ project_sql }} + {% endset %} + {% do run_query(insert_sql) %} + {% endfor %} + {% endif %} + + {{ run_hooks(post_hooks) }} + + {{ return({'relations': [target_relation]}) }} + + {%- endmaterialization -%} +{% endmacro %} diff --git a/models/base/google/access_control/information_schema_object_privileges.sql b/models/base/google/access_control/information_schema_object_privileges.sql index 125115e..3dd14c9 100644 --- a/models/base/google/access_control/information_schema_object_privileges.sql +++ b/models/base/google/access_control/information_schema_object_privileges.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-object-privileges -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.OBJECT_PRIVILEGES view, you need following Identity and Access Management (IAM) permissions: @@ -6,31 +7,5 @@ bigquery.tables.getIamPolicy for tables and views. For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT object_catalog, object_schema, object_name, object_type, privilege_type, grantee - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`OBJECT_PRIVILEGES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -object_catalog, -object_schema, -object_name, -object_type, -privilege_type, -grantee +SELECT object_catalog, object_schema, object_name, object_type, privilege_type, grantee FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`OBJECT_PRIVILEGES` -{%- endif %} -) - -SELECT -object_catalog, -object_schema, -object_name, -object_type, -privilege_type, -grantee, -FROM -base diff --git a/models/base/google/bi_engine/information_schema_bi_capacities.sql b/models/base/google/bi_engine/information_schema_bi_capacities.sql index 1a0b1ef..8dc3539 100644 --- a/models/base/google/bi_engine/information_schema_bi_capacities.sql +++ b/models/base/google/bi_engine/information_schema_bi_capacities.sql @@ -1,28 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-bi-capacities -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT project_id, project_number, bi_capacity_name, size, preferred_tables - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITIES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -project_id, -project_number, -bi_capacity_name, -size, -preferred_tables +SELECT project_id, project_number, bi_capacity_name, size, preferred_tables FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITIES` -{%- endif %} -) - -SELECT -project_id, -project_number, -bi_capacity_name, -size, -preferred_tables, -FROM -base diff --git a/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql b/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql index 3176323..a112536 100644 --- a/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql +++ b/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql @@ -1,32 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-bi-capacity-changes -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT change_timestamp, project_id, project_number, bi_capacity_name, size, user_email, preferred_tables - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITY_CHANGES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -change_timestamp, -project_id, -project_number, -bi_capacity_name, -size, -user_email, -preferred_tables +SELECT change_timestamp, project_id, project_number, bi_capacity_name, size, user_email, preferred_tables FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITY_CHANGES` -{%- endif %} -) - -SELECT -change_timestamp, -project_id, -project_number, -bi_capacity_name, -size, -user_email, -preferred_tables, -FROM -base diff --git a/models/base/google/configuration/information_schema_effective_project_options.sql b/models/base/google/configuration/information_schema_effective_project_options.sql index d940b4c..dbc24cf 100644 --- a/models/base/google/configuration/information_schema_effective_project_options.sql +++ b/models/base/google/configuration/information_schema_effective_project_options.sql @@ -6,21 +6,5 @@ roles/bigquery.jobUser For more information about granular BigQuery permissions, see roles and permissions. -#} -WITH base AS ( - SELECT -option_name, -option_description, -option_type, -option_set_level, -option_set_on_id -FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`EFFECTIVE_PROJECT_OPTIONS` -) - -SELECT -option_name, -option_description, -option_type, -option_set_level, -option_set_on_id, -FROM -base +SELECT option_name, option_description, option_type, option_set_level, option_set_on_id +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`EFFECTIVE_PROJECT_OPTIONS` \ No newline at end of file diff --git a/models/base/google/configuration/information_schema_organization_options.sql b/models/base/google/configuration/information_schema_organization_options.sql index 27a5822..8f1d983 100644 --- a/models/base/google/configuration/information_schema_organization_options.sql +++ b/models/base/google/configuration/information_schema_organization_options.sql @@ -7,19 +7,5 @@ roles/bigquery.jobUser For more information about granular BigQuery permissions, see roles and permissions. -#} -WITH base AS ( - SELECT -option_name, -option_description, -option_type, -option_value -FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS` -) - -SELECT -option_name, -option_description, -option_type, -option_value, -FROM -base +SELECT option_name, option_description, option_type, option_value +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS` \ No newline at end of file diff --git a/models/base/google/configuration/information_schema_organization_options_changes.sql b/models/base/google/configuration/information_schema_organization_options_changes.sql index 7d7cab2..6548d88 100644 --- a/models/base/google/configuration/information_schema_organization_options_changes.sql +++ b/models/base/google/configuration/information_schema_organization_options_changes.sql @@ -15,21 +15,5 @@ -#} -WITH base AS ( - SELECT -update_time, -username, -updated_options, -project_id, -project_number -FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS_CHANGES` -) - -SELECT -update_time, -username, -updated_options, -project_id, -project_number, -FROM -base +SELECT update_time, username, updated_options, project_id, project_number +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS_CHANGES` \ No newline at end of file diff --git a/models/base/google/configuration/information_schema_project_options.sql b/models/base/google/configuration/information_schema_project_options.sql index ab55e85..3e57c9c 100644 --- a/models/base/google/configuration/information_schema_project_options.sql +++ b/models/base/google/configuration/information_schema_project_options.sql @@ -7,19 +7,5 @@ roles/bigquery.jobUser For more information about granular BigQuery permissions, see roles and permissions. -#} -WITH base AS ( - SELECT -option_name, -option_description, -option_type, -option_value -FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS` -) - -SELECT -option_name, -option_description, -option_type, -option_value, -FROM -base +SELECT option_name, option_description, option_type, option_value +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS` \ No newline at end of file diff --git a/models/base/google/configuration/information_schema_project_options_changes.sql b/models/base/google/configuration/information_schema_project_options_changes.sql index 66ae79b..2c22dd8 100644 --- a/models/base/google/configuration/information_schema_project_options_changes.sql +++ b/models/base/google/configuration/information_schema_project_options_changes.sql @@ -5,21 +5,5 @@ IAM role roles/bigquery.admin includes the permissions that you need to create a configuration.For more information about granular BigQuery permissions, see roles and permissions. -#} -WITH base AS ( - SELECT -update_time, -username, -updated_options, -project_id, -project_number -FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS_CHANGES` -) - -SELECT -update_time, -username, -updated_options, -project_id, -project_number, -FROM -base +SELECT update_time, username, updated_options, project_id, project_number +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS_CHANGES` \ No newline at end of file diff --git a/models/base/google/datasets/information_schema_schemata.sql b/models/base/google/datasets/information_schema_schemata.sql index cd28dd0..13545ee 100644 --- a/models/base/google/datasets/information_schema_schemata.sql +++ b/models/base/google/datasets/information_schema_schemata.sql @@ -1,34 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT catalog_name, schema_name, schema_owner, creation_time, last_modified_time, location, ddl, default_collation_name - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -catalog_name, -schema_name, -schema_owner, -creation_time, -last_modified_time, -location, -ddl, -default_collation_name +SELECT catalog_name, schema_name, schema_owner, creation_time, last_modified_time, location, ddl, default_collation_name FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` -{%- endif %} -) - -SELECT -catalog_name, -schema_name, -schema_owner, -creation_time, -last_modified_time, -location, -ddl, -default_collation_name, -FROM -base diff --git a/models/base/google/datasets/information_schema_schemata_links.sql b/models/base/google/datasets/information_schema_schemata_links.sql index d32e54e..adc5ee0 100644 --- a/models/base/google/datasets/information_schema_schemata_links.sql +++ b/models/base/google/datasets/information_schema_schemata_links.sql @@ -1,32 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata-links -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT catalog_name, schema_name, linked_schema_catalog_number, linked_schema_catalog_name, linked_schema_name, linked_schema_creation_time, linked_schema_org_display_name - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_LINKS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -catalog_name, -schema_name, -linked_schema_catalog_number, -linked_schema_catalog_name, -linked_schema_name, -linked_schema_creation_time, -linked_schema_org_display_name +SELECT catalog_name, schema_name, linked_schema_catalog_number, linked_schema_catalog_name, linked_schema_name, linked_schema_creation_time, linked_schema_org_display_name FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_LINKS` -{%- endif %} -) - -SELECT -catalog_name, -schema_name, -linked_schema_catalog_number, -linked_schema_catalog_name, -linked_schema_name, -linked_schema_creation_time, -linked_schema_org_display_name, -FROM -base diff --git a/models/base/google/datasets/information_schema_schemata_options.sql b/models/base/google/datasets/information_schema_schemata_options.sql index b0384ae..75922aa 100644 --- a/models/base/google/datasets/information_schema_schemata_options.sql +++ b/models/base/google/datasets/information_schema_schemata_options.sql @@ -1,28 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata-options -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT catalog_name, schema_name, option_name, option_type, option_value - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_OPTIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -catalog_name, -schema_name, -option_name, -option_type, -option_value +SELECT catalog_name, schema_name, option_name, option_type, option_value FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_OPTIONS` -{%- endif %} -) - -SELECT -catalog_name, -schema_name, -option_name, -option_type, -option_value, -FROM -base diff --git a/models/base/google/datasets/information_schema_schemata_replicas.sql b/models/base/google/datasets/information_schema_schemata_replicas.sql index e34f0e9..fc8eb66 100644 --- a/models/base/google/datasets/information_schema_schemata_replicas.sql +++ b/models/base/google/datasets/information_schema_schemata_replicas.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-schemata-replicas -#} {# Required role/permissions: @@ -20,39 +21,5 @@ roles. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT catalog_name, schema_name, replica_name, location, replica_primary_assigned, replica_primary_assignment_complete, creation_time, creation_complete, replication_time, sync_status - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_REPLICAS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -catalog_name, -schema_name, -replica_name, -location, -replica_primary_assigned, -replica_primary_assignment_complete, -creation_time, -creation_complete, -replication_time, -sync_status +SELECT catalog_name, schema_name, replica_name, location, replica_primary_assigned, replica_primary_assignment_complete, creation_time, creation_complete, replication_time, sync_status FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_REPLICAS` -{%- endif %} -) - -SELECT -catalog_name, -schema_name, -replica_name, -location, -replica_primary_assigned, -replica_primary_assignment_complete, -creation_time, -creation_complete, -replication_time, -sync_status, -FROM -base diff --git a/models/base/google/datasets/information_schema_schemata_replicas_by_failover_reservation.sql b/models/base/google/datasets/information_schema_schemata_replicas_by_failover_reservation.sql index 90f48fc..476bb47 100644 --- a/models/base/google/datasets/information_schema_schemata_replicas_by_failover_reservation.sql +++ b/models/base/google/datasets/information_schema_schemata_replicas_by_failover_reservation.sql @@ -20,35 +20,5 @@ roles. -#} -WITH base AS ( - SELECT -failover_reservation_project_id, -failover_reservation_name, -catalog_name, -schema_name, -replica_name, -location, -replica_primary_assigned, -replica_primary_assignment_complete, -creation_time, -creation_complete, -replication_time, -sync_status -FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_REPLICAS_BY_FAILOVER_RESERVATION` -) - -SELECT -failover_reservation_project_id, -failover_reservation_name, -catalog_name, -schema_name, -replica_name, -location, -replica_primary_assigned, -replica_primary_assignment_complete, -creation_time, -creation_complete, -replication_time, -sync_status, -FROM -base +SELECT failover_reservation_project_id, failover_reservation_name, catalog_name, schema_name, replica_name, location, replica_primary_assigned, replica_primary_assignment_complete, creation_time, creation_complete, replication_time, sync_status +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_REPLICAS_BY_FAILOVER_RESERVATION` \ No newline at end of file diff --git a/models/base/google/datasets/information_schema_shared_dataset_usage.sql b/models/base/google/datasets/information_schema_shared_dataset_usage.sql index 0a774e2..d616b33 100644 --- a/models/base/google/datasets/information_schema_shared_dataset_usage.sql +++ b/models/base/google/datasets/information_schema_shared_dataset_usage.sql @@ -1,50 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-shared-dataset-usage -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT project_id, dataset_id, table_id, data_exchange_id, listing_id, job_start_time, job_end_time, job_id, job_project_number, job_location, linked_project_number, linked_dataset_id, subscriber_org_number, subscriber_org_display_name, num_rows_processed, total_bytes_processed - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SHARED_DATASET_USAGE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -project_id, -dataset_id, -table_id, -data_exchange_id, -listing_id, -job_start_time, -job_end_time, -job_id, -job_project_number, -job_location, -linked_project_number, -linked_dataset_id, -subscriber_org_number, -subscriber_org_display_name, -num_rows_processed, -total_bytes_processed +SELECT project_id, dataset_id, table_id, data_exchange_id, listing_id, job_start_time, job_end_time, job_id, job_project_number, job_location, linked_project_number, linked_dataset_id, subscriber_org_number, subscriber_org_display_name, num_rows_processed, total_bytes_processed FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SHARED_DATASET_USAGE` -{%- endif %} -) - -SELECT -project_id, -dataset_id, -table_id, -data_exchange_id, -listing_id, -job_start_time, -job_end_time, -job_id, -job_project_number, -job_location, -linked_project_number, -linked_dataset_id, -subscriber_org_number, -subscriber_org_display_name, -num_rows_processed, -total_bytes_processed, -FROM -base diff --git a/models/base/google/jobs/information_schema_jobs.sql b/models/base/google/jobs/information_schema_jobs.sql index 9c63abe..d812669 100644 --- a/models/base/google/jobs/information_schema_jobs.sql +++ b/models/base/google/jobs/information_schema_jobs.sql @@ -1,5 +1,6 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs -#} -{# Required role/permissions: +{# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS view, @@ -15,88 +16,5 @@ -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT - STRUCT( - bi_engine_statistics.bi_engine_mode AS bi_engine_mode, - bi_engine_statistics.bi_engine_reasons AS bi_engine_reasons, - bi_engine_statistics.acceleration_mode AS acceleration_mode) AS bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -edition, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -job_creation_reason, -query_info +SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS` -{%- endif %} -) - -SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -edition, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -job_creation_reason, -query_info, -FROM -base diff --git a/models/base/google/jobs/information_schema_jobs_by_folder.sql b/models/base/google/jobs/information_schema_jobs_by_folder.sql index d26e9aa..2480acd 100644 --- a/models/base/google/jobs/information_schema_jobs_by_folder.sql +++ b/models/base/google/jobs/information_schema_jobs_by_folder.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-folder -#} {# Required role/permissions: @@ -15,85 +16,5 @@ -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_FOLDER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -edition, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -job_creation_reason, -query_info +SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_FOLDER` -{%- endif %} -) - -SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -edition, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -job_creation_reason, -query_info, -FROM -base diff --git a/models/base/google/jobs/information_schema_jobs_by_organization.sql b/models/base/google/jobs/information_schema_jobs_by_organization.sql index da60b58..b1aa626 100644 --- a/models/base/google/jobs/information_schema_jobs_by_organization.sql +++ b/models/base/google/jobs/information_schema_jobs_by_organization.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-organization -#} {# Required role/permissions: @@ -15,85 +16,5 @@ -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_ORGANIZATION` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -edition, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -job_creation_reason, -query_info +SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_ORGANIZATION` -{%- endif %} -) - -SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -edition, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -job_creation_reason, -query_info, -FROM -base diff --git a/models/base/google/jobs/information_schema_jobs_by_project.sql b/models/base/google/jobs/information_schema_jobs_by_project.sql index 57d6447..667a3f2 100644 --- a/models/base/google/jobs/information_schema_jobs_by_project.sql +++ b/models/base/google/jobs/information_schema_jobs_by_project.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs -#} {# Required role/permissions: @@ -15,85 +16,5 @@ -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_PROJECT` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -edition, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -job_creation_reason, -query_info +SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_PROJECT` -{%- endif %} -) - -SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -edition, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -job_creation_reason, -query_info, -FROM -base diff --git a/models/base/google/jobs/information_schema_jobs_by_user.sql b/models/base/google/jobs/information_schema_jobs_by_user.sql index 2733610..0ed7e69 100644 --- a/models/base/google/jobs/information_schema_jobs_by_user.sql +++ b/models/base/google/jobs/information_schema_jobs_by_user.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-user -#} {# Required role/permissions: @@ -15,87 +16,5 @@ -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_USER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -dml_statistics, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -edition, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -job_creation_reason, -query_info +SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_USER` -{%- endif %} -) - -SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -dml_statistics, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -edition, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -job_creation_reason, -query_info, -FROM -base diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql index f671e34..7314c21 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the project. @@ -8,57 +9,5 @@ BigQuery Admin For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, edition, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -period_start, -period_slot_ms, -period_shuffle_ram_usage_ratio, -project_id, -project_number, -user_email, -job_id, -job_type, -statement_type, -job_creation_time, -job_start_time, -job_end_time, -state, -reservation_id, -edition, -total_bytes_processed, -error_result, -cache_hit, -period_estimated_runnable_units +SELECT period_start, period_slot_ms, project_id, project_number, user_email, job_id, job_type, statement_type, priority, parent_job_id, job_creation_time, job_start_time, job_end_time, state, reservation_id, edition, total_bytes_billed, total_bytes_processed, error_result, cache_hit, period_shuffle_ram_usage_ratio, period_estimated_runnable_units, transaction_id FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE` -{%- endif %} -) - -SELECT -period_start, -period_slot_ms, -period_shuffle_ram_usage_ratio, -project_id, -project_number, -user_email, -job_id, -job_type, -statement_type, -job_creation_time, -job_start_time, -job_end_time, -state, -reservation_id, -edition, -total_bytes_processed, -error_result, -cache_hit, -period_estimated_runnable_units, -FROM -base diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline.yml b/models/base/google/jobs_timeline/information_schema_jobs_timeline.yml index 8d5698a..275db87 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline.yml +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline.yml @@ -9,9 +9,6 @@ models: - name: period_slot_ms description: Slot milliseconds consumed in this period. type: INTEGER - - name: period_shuffle_ram_usage_ratio - description: Shuffle usage ratio in the selected time period. - type: FLOAT - name: project_id description: (Clustering column) ID of the project. type: STRING @@ -34,6 +31,13 @@ models: description: "The type of query statement, if valid. For example,\n SELECT,\ \ INSERT, UPDATE, or\n DELETE." type: STRING + - name: priority + description: "The priority of this job. Valid values include INTERACTIVE and\n\ + \ BATCH." + type: STRING + - name: parent_job_id + description: ID of the parent job, if any. + type: STRING - name: job_creation_time description: "(Partitioning column) Creation time of this job. Partitioning\n\ \ is based on the UTC time of this timestamp." @@ -56,6 +60,12 @@ models: description: The edition associated with the reservation assigned to this job. For more information about editions, see Introduction to BigQuery editions. type: STRING + - name: total_bytes_billed + description: "If the project is configured to use on-demand\n pricing,\ + \ then this field contains the total bytes billed for the\n job. If the\ + \ project is configured to use flat-rate\n pricing, then you are not\ + \ billed for bytes and this field is\n informational only." + type: INTEGER - name: total_bytes_processed description: Total bytes processed by the job. type: INTEGER @@ -65,8 +75,14 @@ models: - name: cache_hit description: Whether the query results of this job were from a cache. type: BOOLEAN + - name: period_shuffle_ram_usage_ratio + description: Shuffle usage ratio in the selected time period. + type: FLOAT - name: period_estimated_runnable_units description: Units of work that can be scheduled immediately in this period. Additional slots for these units of work accelerate your query, provided no other query in the reservation needs additional slots. type: INTEGER + - name: transaction_id + description: "ID of the transaction\n in which this job ran, if any. (Preview)" + type: STRING diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql index 20c418a..8f7bf6d 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-folder -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_FOLDER view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the parent @@ -8,59 +9,5 @@ BigQuery Admin For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, folder_numbers, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, edition, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_FOLDER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -period_start, -period_slot_ms, -period_shuffle_ram_usage_ratio, -project_id, -project_number, -folder_numbers, -user_email, -job_id, -job_type, -statement_type, -job_creation_time, -job_start_time, -job_end_time, -state, -reservation_id, -edition, -total_bytes_processed, -error_result, -cache_hit, -period_estimated_runnable_units +SELECT period_start, period_slot_ms, project_id, project_number, folder_numbers, user_email, job_id, job_type, statement_type, priority, parent_job_id, job_creation_time, job_start_time, job_end_time, state, reservation_id, edition, total_bytes_billed, total_bytes_processed, error_result, cache_hit, period_shuffle_ram_usage_ratio, period_estimated_runnable_units, transaction_id FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_FOLDER` -{%- endif %} -) - -SELECT -period_start, -period_slot_ms, -period_shuffle_ram_usage_ratio, -project_id, -project_number, -folder_numbers, -user_email, -job_id, -job_type, -statement_type, -job_creation_time, -job_start_time, -job_end_time, -state, -reservation_id, -edition, -total_bytes_processed, -error_result, -cache_hit, -period_estimated_runnable_units, -FROM -base diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.yml b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.yml index 76b0ae9..9d30429 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.yml +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.yml @@ -9,9 +9,6 @@ models: - name: period_slot_ms description: Slot milliseconds consumed in this period. type: INTEGER - - name: period_shuffle_ram_usage_ratio - description: Shuffle usage ratio in the selected time period. - type: FLOAT - name: project_id description: (Clustering column) ID of the project. type: STRING @@ -41,6 +38,13 @@ models: description: "The type of query statement, if valid. For example,\n SELECT,\ \ INSERT, UPDATE, or\n DELETE." type: STRING + - name: priority + description: "The priority of this job. Valid values include INTERACTIVE and\n\ + \ BATCH." + type: STRING + - name: parent_job_id + description: ID of the parent job, if any. + type: STRING - name: job_creation_time description: "(Partitioning column) Creation time of this job. Partitioning\n\ \ is based on the UTC time of this timestamp." @@ -63,6 +67,12 @@ models: description: The edition associated with the reservation assigned to this job. For more information about editions, see Introduction to BigQuery editions. type: STRING + - name: total_bytes_billed + description: "If the project is configured to use on-demand\n pricing,\ + \ then this field contains the total bytes billed for the\n job. If the\ + \ project is configured to use flat-rate\n pricing, then you are not\ + \ billed for bytes and this field is\n informational only." + type: INTEGER - name: total_bytes_processed description: Total bytes processed by the job. type: INTEGER @@ -72,8 +82,14 @@ models: - name: cache_hit description: Whether the query results of this job were from a cache. type: BOOLEAN + - name: period_shuffle_ram_usage_ratio + description: Shuffle usage ratio in the selected time period. + type: FLOAT - name: period_estimated_runnable_units description: Units of work that can be scheduled immediately in this period. Additional slots for these units of work accelerate your query, provided no other query in the reservation needs additional slots. type: INTEGER + - name: transaction_id + description: "ID of the transaction\n in which this job ran, if any. (Preview)" + type: STRING diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql index d273e10..255113c 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-organization -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_ORGANIZATION view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the organization. @@ -10,59 +11,5 @@ The JOBS_BY_ORGANIZATION schema table is only available to users with defined Google Cloud organizations.For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, folder_numbers, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, edition, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_ORGANIZATION` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -period_start, -period_slot_ms, -period_shuffle_ram_usage_ratio, -project_id, -project_number, -folder_numbers, -user_email, -job_id, -job_type, -statement_type, -job_creation_time, -job_start_time, -job_end_time, -state, -reservation_id, -edition, -total_bytes_processed, -error_result, -cache_hit, -period_estimated_runnable_units +SELECT period_start, period_slot_ms, project_id, project_number, folder_numbers, user_email, job_id, job_type, statement_type, priority, parent_job_id, job_creation_time, job_start_time, job_end_time, state, reservation_id, edition, total_bytes_billed, total_bytes_processed, error_result, cache_hit, period_shuffle_ram_usage_ratio, period_estimated_runnable_units FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_ORGANIZATION` -{%- endif %} -) - -SELECT -period_start, -period_slot_ms, -period_shuffle_ram_usage_ratio, -project_id, -project_number, -folder_numbers, -user_email, -job_id, -job_type, -statement_type, -job_creation_time, -job_start_time, -job_end_time, -state, -reservation_id, -edition, -total_bytes_processed, -error_result, -cache_hit, -period_estimated_runnable_units, -FROM -base diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.yml b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.yml index 6ec8dbb..3b24ccd 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.yml +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.yml @@ -9,9 +9,6 @@ models: - name: period_slot_ms description: Slot milliseconds consumed in this period. type: INTEGER - - name: period_shuffle_ram_usage_ratio - description: Shuffle usage ratio in the selected time period. - type: FLOAT - name: project_id description: (Clustering column) ID of the project. type: STRING @@ -41,6 +38,13 @@ models: description: "The type of query statement, if valid. For example,\n SELECT,\ \ INSERT, UPDATE, or\n DELETE." type: STRING + - name: priority + description: "The priority of this job. Valid values include INTERACTIVE and\n\ + \ BATCH." + type: STRING + - name: parent_job_id + description: ID of the parent job, if any. + type: STRING - name: job_creation_time description: "(Partitioning column) Creation time of this job. Partitioning\n\ \ is based on the UTC time of this timestamp." @@ -63,6 +67,12 @@ models: description: The edition associated with the reservation assigned to this job. For more information about editions, see Introduction to BigQuery editions. type: STRING + - name: total_bytes_billed + description: "If the project is configured to use on-demand\n pricing,\ + \ then this field contains the total bytes billed for the\n job. If the\ + \ project is configured to use flat-rate\n pricing, then you are not\ + \ billed for bytes. This field is\n not configurable." + type: INTEGER - name: total_bytes_processed description: Total bytes processed by the job. type: INTEGER @@ -72,6 +82,9 @@ models: - name: cache_hit description: Whether the query results of this job were from a cache. type: BOOLEAN + - name: period_shuffle_ram_usage_ratio + description: Shuffle usage ratio in the selected time period. + type: FLOAT - name: period_estimated_runnable_units description: Units of work that can be scheduled immediately in this period. Additional slots for these units of work accelerate your query, provided no other query diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql index 258c266..34f9426 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-user -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_USER view, you need the bigquery.jobs.list Identity and Access Management (IAM) permission for the project. @@ -8,57 +9,5 @@ BigQuery User For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, edition, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_USER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -period_start, -period_slot_ms, -period_shuffle_ram_usage_ratio, -project_id, -project_number, -user_email, -job_id, -job_type, -statement_type, -job_creation_time, -job_start_time, -job_end_time, -state, -reservation_id, -edition, -total_bytes_processed, -error_result, -cache_hit, -period_estimated_runnable_units +SELECT period_start, period_slot_ms, project_id, project_number, user_email, job_id, job_type, statement_type, priority, parent_job_id, job_creation_time, job_start_time, job_end_time, state, reservation_id, edition, total_bytes_billed, total_bytes_processed, error_result, cache_hit, period_shuffle_ram_usage_ratio, period_estimated_runnable_units, transaction_id FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_USER` -{%- endif %} -) - -SELECT -period_start, -period_slot_ms, -period_shuffle_ram_usage_ratio, -project_id, -project_number, -user_email, -job_id, -job_type, -statement_type, -job_creation_time, -job_start_time, -job_end_time, -state, -reservation_id, -edition, -total_bytes_processed, -error_result, -cache_hit, -period_estimated_runnable_units, -FROM -base diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.yml b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.yml index 343658e..c175fed 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.yml +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.yml @@ -9,9 +9,6 @@ models: - name: period_slot_ms description: Slot milliseconds consumed in this period. type: INTEGER - - name: period_shuffle_ram_usage_ratio - description: Shuffle usage ratio in the selected time period. - type: FLOAT - name: project_id description: (Clustering column) ID of the project. type: STRING @@ -34,6 +31,13 @@ models: description: "The type of query statement, if valid. For example,\n SELECT,\ \ INSERT, UPDATE, or\n DELETE." type: STRING + - name: priority + description: "The priority of this job. Valid values include INTERACTIVE and\n\ + \ BATCH." + type: STRING + - name: parent_job_id + description: ID of the parent job, if any. + type: STRING - name: job_creation_time description: "(Partitioning column) Creation time of this job. Partitioning\n\ \ is based on the UTC time of this timestamp." @@ -56,6 +60,12 @@ models: description: The edition associated with the reservation assigned to this job. For more information about editions, see Introduction to BigQuery editions. type: STRING + - name: total_bytes_billed + description: "If the project is configured to use on-demand\n pricing,\ + \ then this field contains the total bytes billed for the\n job. If the\ + \ project is configured to use flat-rate\n pricing, then you are not\ + \ billed for bytes and this field is\n informational only." + type: INTEGER - name: total_bytes_processed description: Total bytes processed by the job. type: INTEGER @@ -65,8 +75,14 @@ models: - name: cache_hit description: Whether the query results of this job were from a cache. type: BOOLEAN + - name: period_shuffle_ram_usage_ratio + description: Shuffle usage ratio in the selected time period. + type: FLOAT - name: period_estimated_runnable_units description: Units of work that can be scheduled immediately in this period. Additional slots for these units of work accelerate your query, provided no other query in the reservation needs additional slots. type: INTEGER + - name: transaction_id + description: "ID of the transaction\n in which this job ran, if any. (Preview)" + type: STRING diff --git a/models/base/google/recommendations_and_insights/information_schema_insights.sql b/models/base/google/recommendations_and_insights/information_schema_insights.sql index 9c4b5a1..5f30e24 100644 --- a/models/base/google/recommendations_and_insights/information_schema_insights.sql +++ b/models/base/google/recommendations_and_insights/information_schema_insights.sql @@ -1,44 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-insights -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT insight_id, insight_type, subtype, project_id, project_number, description, last_updated_time, category, target_resources, state, severity, associated_recommendation_ids, additional_details - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`INSIGHTS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -insight_id, -insight_type, -subtype, -project_id, -project_number, -description, -last_updated_time, -category, -target_resources, -state, -severity, -associated_recommendation_ids, -additional_details +SELECT insight_id, insight_type, subtype, project_id, project_number, description, last_updated_time, category, target_resources, state, severity, associated_recommendation_ids, additional_details FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`INSIGHTS` -{%- endif %} -) - -SELECT -insight_id, -insight_type, -subtype, -project_id, -project_number, -description, -last_updated_time, -category, -target_resources, -state, -severity, -associated_recommendation_ids, -additional_details, -FROM -base diff --git a/models/base/google/recommendations_and_insights/information_schema_recommendations.sql b/models/base/google/recommendations_and_insights/information_schema_recommendations.sql index 924e630..3be4692 100644 --- a/models/base/google/recommendations_and_insights/information_schema_recommendations.sql +++ b/models/base/google/recommendations_and_insights/information_schema_recommendations.sql @@ -1,44 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-recommendations -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT recommendation_id, recommender, subtype, project_id, project_number, description, last_updated_time, target_resources, state, primary_impact, priority, associated_insight_ids, additional_details - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RECOMMENDATIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -recommendation_id, -recommender, -subtype, -project_id, -project_number, -description, -last_updated_time, -target_resources, -state, -primary_impact, -priority, -associated_insight_ids, -additional_details +SELECT recommendation_id, recommender, subtype, project_id, project_number, description, last_updated_time, target_resources, state, primary_impact, priority, associated_insight_ids, additional_details FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RECOMMENDATIONS` -{%- endif %} -) - -SELECT -recommendation_id, -recommender, -subtype, -project_id, -project_number, -description, -last_updated_time, -target_resources, -state, -primary_impact, -priority, -associated_insight_ids, -additional_details, -FROM -base diff --git a/models/base/google/recommendations_and_insights/information_schema_recommendations_by_organization.sql b/models/base/google/recommendations_and_insights/information_schema_recommendations_by_organization.sql index aa075cb..2fb2683 100644 --- a/models/base/google/recommendations_and_insights/information_schema_recommendations_by_organization.sql +++ b/models/base/google/recommendations_and_insights/information_schema_recommendations_by_organization.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-recommendations-by-org -#} {# Required role/permissions: To view recommendations with the INFORMATION_SCHEMA.RECOMMENDATIONS_BY_ORGANIZATION view, you must have the @@ -12,45 +13,5 @@ Materialized view recommendations permissions Role recommendations for datasets permissions -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT recommendation_id, recommender, subtype, project_id, project_number, description, last_updated_time, target_resources, state, primary_impact, priority, associated_insight_ids, additional_details - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RECOMMENDATIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -recommendation_id, -recommender, -subtype, -project_id, -project_number, -description, -last_updated_time, -target_resources, -state, -primary_impact, -priority, -associated_insight_ids, -additional_details +SELECT recommendation_id, recommender, subtype, project_id, project_number, description, last_updated_time, target_resources, state, primary_impact, priority, associated_insight_ids, additional_details FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RECOMMENDATIONS` -{%- endif %} -) - -SELECT -recommendation_id, -recommender, -subtype, -project_id, -project_number, -description, -last_updated_time, -target_resources, -state, -primary_impact, -priority, -associated_insight_ids, -additional_details, -FROM -base diff --git a/models/base/google/reservations/information_schema_assignment_changes.sql b/models/base/google/reservations/information_schema_assignment_changes.sql index 3e84757..742ae42 100644 --- a/models/base/google/reservations/information_schema_assignment_changes.sql +++ b/models/base/google/reservations/information_schema_assignment_changes.sql @@ -1,42 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-assignments-changes -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT change_timestamp, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type, action, user_email, state - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENT_CHANGES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -change_timestamp, -project_id, -project_number, -assignment_id, -reservation_name, -job_type, -assignee_id, -assignee_number, -assignee_type, -action, -user_email, -state +SELECT change_timestamp, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type, action, user_email, state FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENT_CHANGES` -{%- endif %} -) - -SELECT -change_timestamp, -project_id, -project_number, -assignment_id, -reservation_name, -job_type, -assignee_id, -assignee_number, -assignee_type, -action, -user_email, -state, -FROM -base diff --git a/models/base/google/reservations/information_schema_assignments.sql b/models/base/google/reservations/information_schema_assignments.sql index 28cbfc9..7e65b35 100644 --- a/models/base/google/reservations/information_schema_assignments.sql +++ b/models/base/google/reservations/information_schema_assignments.sql @@ -1,36 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-assignments -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT ddl, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENTS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -ddl, -project_id, -project_number, -assignment_id, -reservation_name, -job_type, -assignee_id, -assignee_number, -assignee_type +SELECT ddl, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENTS` -{%- endif %} -) - -SELECT -ddl, -project_id, -project_number, -assignment_id, -reservation_name, -job_type, -assignee_id, -assignee_number, -assignee_type, -FROM -base diff --git a/models/base/google/reservations/information_schema_capacity_commitment_changes.sql b/models/base/google/reservations/information_schema_capacity_commitment_changes.sql index 6b438e3..b867aef 100644 --- a/models/base/google/reservations/information_schema_capacity_commitment_changes.sql +++ b/models/base/google/reservations/information_schema_capacity_commitment_changes.sql @@ -1,48 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-capacity-commitment-changes -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT change_timestamp, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, action, user_email, commitment_start_time, commitment_end_time, failure_status, renewal_plan, edition, is_flat_rate - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENT_CHANGES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -change_timestamp, -project_id, -project_number, -capacity_commitment_id, -commitment_plan, -state, -slot_count, -action, -user_email, -commitment_start_time, -commitment_end_time, -failure_status, -renewal_plan, -edition, -is_flat_rate +SELECT change_timestamp, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, action, user_email, commitment_start_time, commitment_end_time, failure_status, renewal_plan, edition, is_flat_rate FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENT_CHANGES` -{%- endif %} -) - -SELECT -change_timestamp, -project_id, -project_number, -capacity_commitment_id, -commitment_plan, -state, -slot_count, -action, -user_email, -commitment_start_time, -commitment_end_time, -failure_status, -renewal_plan, -edition, -is_flat_rate, -FROM -base diff --git a/models/base/google/reservations/information_schema_capacity_commitments.sql b/models/base/google/reservations/information_schema_capacity_commitments.sql index c25ff0a..c50a12f 100644 --- a/models/base/google/reservations/information_schema_capacity_commitments.sql +++ b/models/base/google/reservations/information_schema_capacity_commitments.sql @@ -1,38 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-capacity-commitments -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT ddl, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, edition, is_flat_rate, renewal_plan - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENTS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -ddl, -project_id, -project_number, -capacity_commitment_id, -commitment_plan, -state, -slot_count, -edition, -is_flat_rate, -renewal_plan +SELECT ddl, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, edition, is_flat_rate, renewal_plan FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENTS` -{%- endif %} -) - -SELECT -ddl, -project_id, -project_number, -capacity_commitment_id, -commitment_plan, -state, -slot_count, -edition, -is_flat_rate, -renewal_plan, -FROM -base diff --git a/models/base/google/reservations/information_schema_reservation_changes.sql b/models/base/google/reservations/information_schema_reservation_changes.sql index 75fc9ea..77e6088 100644 --- a/models/base/google/reservations/information_schema_reservation_changes.sql +++ b/models/base/google/reservations/information_schema_reservation_changes.sql @@ -1,40 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservation-changes -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT change_timestamp, project_id, project_number, reservation_name, ignore_idle_slots, action, slot_capacity, user_email, target_job_concurrency, autoscale, edition - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATION_CHANGES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -change_timestamp, -project_id, -project_number, -reservation_name, -ignore_idle_slots, -action, -slot_capacity, -user_email, -target_job_concurrency, -autoscale, -edition +SELECT change_timestamp, project_id, project_number, reservation_name, ignore_idle_slots, action, slot_capacity, user_email, target_job_concurrency, autoscale, edition FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATION_CHANGES` -{%- endif %} -) - -SELECT -change_timestamp, -project_id, -project_number, -reservation_name, -ignore_idle_slots, -action, -slot_capacity, -user_email, -target_job_concurrency, -autoscale, -edition, -FROM -base diff --git a/models/base/google/reservations/information_schema_reservations.sql b/models/base/google/reservations/information_schema_reservations.sql index a1382a7..235cdfd 100644 --- a/models/base/google/reservations/information_schema_reservations.sql +++ b/models/base/google/reservations/information_schema_reservations.sql @@ -1,36 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservations -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT ddl, project_id, project_number, reservation_name, ignore_idle_slots, slot_capacity, target_job_concurrency, autoscale, edition - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -ddl, -project_id, -project_number, -reservation_name, -ignore_idle_slots, -slot_capacity, -target_job_concurrency, -autoscale, -edition +SELECT ddl, project_id, project_number, reservation_name, ignore_idle_slots, slot_capacity, target_job_concurrency, autoscale, edition FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS` -{%- endif %} -) - -SELECT -ddl, -project_id, -project_number, -reservation_name, -ignore_idle_slots, -slot_capacity, -target_job_concurrency, -autoscale, -edition, -FROM -base diff --git a/models/base/google/reservations/information_schema_reservations_timeline.sql b/models/base/google/reservations/information_schema_reservations_timeline.sql index 1c74fd7..ccf4f37 100644 --- a/models/base/google/reservations/information_schema_reservations_timeline.sql +++ b/models/base/google/reservations/information_schema_reservations_timeline.sql @@ -1,36 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservation-timeline -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT period_start, project_id, project_number, reservation_name, ignore_idle_slots, slots_assigned, slots_max_assigned, autoscale, reservation_id - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS_TIMELINE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -period_start, -project_id, -project_number, -reservation_name, -ignore_idle_slots, -slots_assigned, -slots_max_assigned, -autoscale, -reservation_id +SELECT autoscale, edition, ignore_idle_slots, period_start, project_id, project_number, reservation_id, reservation_name, slots_assigned, slots_max_assigned FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS_TIMELINE` -{%- endif %} -) - -SELECT -period_start, -project_id, -project_number, -reservation_name, -ignore_idle_slots, -slots_assigned, -slots_max_assigned, -autoscale, -reservation_id, -FROM -base diff --git a/models/base/google/reservations/information_schema_reservations_timeline.yml b/models/base/google/reservations/information_schema_reservations_timeline.yml index 004f8ed..12a4bfa 100644 --- a/models/base/google/reservations/information_schema_reservations_timeline.yml +++ b/models/base/google/reservations/information_schema_reservations_timeline.yml @@ -3,6 +3,23 @@ models: - name: information_schema_reservations_timeline description: dataset details with related information columns: + - name: autoscale + description: "Information about the autoscale capacity of the reservation. Fields\ + \ include the following:\n\ncurrent_slots: the number of slots added to the\ + \ reservation by autoscaling.\n \nNote: After users reduce max_slots,\ + \ it may take a while before it can be propagated,\n so\ + \ current_slots may stay in the original value and could be larger than max_slots\n\ + \ for that brief period (less than one minute).\n \ + \ \n\nmax_slots: the maximum number of slots that could be added to the\ + \ reservation by autoscaling." + type: STRUCT + - name: edition + description: The edition associated with this reservation. For more information + about editions, see Introduction to BigQuery editions. + type: STRING + - name: ignore_idle_slots + description: False if slot sharing is enabled, otherwise true. + type: BOOL - name: period_start description: Start time of this one-minute period. type: TIMESTAMP @@ -12,12 +29,12 @@ models: - name: project_number description: Number of the project. type: INTEGER + - name: reservation_id + description: For joining with the jobs_timeline table. This is of the form project_id:location.reservation_name. + type: STRING - name: reservation_name description: The name of the reservation. type: STRING - - name: ignore_idle_slots - description: False if slot sharing is enabled, otherwise true. - type: BOOL - name: slots_assigned description: The number of slots assigned to this reservation. type: INTEGER @@ -27,16 +44,3 @@ models: \ otherwise this is the total number of slots in all capacity commitments in\ \ the admin project." type: INTEGER - - name: autoscale - description: "Information about the autoscale capacity of the reservation. Fields\ - \ include the following:\n\ncurrent_slots: the number of slots added to the\ - \ reservation by autoscaling.\n \nNote: After users reduce max_slots,\ - \ it may take a while before it can be propagated,\n so\ - \ current_slots may stay in the original value and could be larger than max_slots\n\ - \ for that brief period (less than one minute).\n \ - \ \n\nmax_slots: the maximum number of slots that could be added to the\ - \ reservation by autoscaling." - type: STRUCT - - name: reservation_id - description: For joining with the jobs_timeline table. This is of the form project_id:location.reservation_name. - type: STRING diff --git a/models/base/google/routines/information_schema_parameters.sql b/models/base/google/routines/information_schema_parameters.sql index 8ac88ba..f076386 100644 --- a/models/base/google/routines/information_schema_parameters.sql +++ b/models/base/google/routines/information_schema_parameters.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-parameters -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.PARAMETERS view, you need the following Identity and Access Management (IAM) permissions: @@ -11,39 +12,5 @@ roles/bigquery.dataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT specific_catalog, specific_schema, specific_name, ordinal_position, parameter_mode, is_result, parameter_name, data_type, parameter_default, is_aggregate - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PARAMETERS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -specific_catalog, -specific_schema, -specific_name, -ordinal_position, -parameter_mode, -is_result, -parameter_name, -data_type, -parameter_default, -is_aggregate +SELECT specific_catalog, specific_schema, specific_name, ordinal_position, parameter_mode, is_result, parameter_name, data_type, parameter_default, is_aggregate FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PARAMETERS` -{%- endif %} -) - -SELECT -specific_catalog, -specific_schema, -specific_name, -ordinal_position, -parameter_mode, -is_result, -parameter_name, -data_type, -parameter_default, -is_aggregate, -FROM -base diff --git a/models/base/google/routines/information_schema_routine_options.sql b/models/base/google/routines/information_schema_routine_options.sql index ad7062a..7cee6ea 100644 --- a/models/base/google/routines/information_schema_routine_options.sql +++ b/models/base/google/routines/information_schema_routine_options.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-routine-options -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.ROUTINE_OPTIONS view, you need the following Identity and Access Management (IAM) permissions: @@ -11,31 +12,5 @@ roles/bigquery.dataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT specific_catalog, specific_schema, specific_name, option_name, option_type, option_value - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINE_OPTIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -specific_catalog, -specific_schema, -specific_name, -option_name, -option_type, -option_value +SELECT specific_catalog, specific_schema, specific_name, option_name, option_type, option_value FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINE_OPTIONS` -{%- endif %} -) - -SELECT -specific_catalog, -specific_schema, -specific_name, -option_name, -option_type, -option_value, -FROM -base diff --git a/models/base/google/routines/information_schema_routines.sql b/models/base/google/routines/information_schema_routines.sql index ec24211..fdeb4a3 100644 --- a/models/base/google/routines/information_schema_routines.sql +++ b/models/base/google/routines/information_schema_routines.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-routines -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.ROUTINES view, you need the following Identity and Access Management (IAM) permissions: @@ -11,53 +12,5 @@ roles/bigquery.dataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT specific_catalog, specific_schema, specific_name, routine_catalog, routine_schema, routine_name, routine_type, data_type, routine_body, routine_definition, external_language, is_deterministic, security_type, created, last_altered, ddl, connection - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -specific_catalog, -specific_schema, -specific_name, -routine_catalog, -routine_schema, -routine_name, -routine_type, -data_type, -routine_body, -routine_definition, -external_language, -is_deterministic, -security_type, -created, -last_altered, -ddl, -connection +SELECT specific_catalog, specific_schema, specific_name, routine_catalog, routine_schema, routine_name, routine_type, data_type, routine_body, routine_definition, external_language, is_deterministic, security_type, created, last_altered, ddl, connection FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINES` -{%- endif %} -) - -SELECT -specific_catalog, -specific_schema, -specific_name, -routine_catalog, -routine_schema, -routine_name, -routine_type, -data_type, -routine_body, -routine_definition, -external_language, -is_deterministic, -security_type, -created, -last_altered, -ddl, -connection, -FROM -base diff --git a/models/base/google/search_indexes/information_schema_search_index_columns.sql b/models/base/google/search_indexes/information_schema_search_index_columns.sql index 303f274..1c55b66 100644 --- a/models/base/google/search_indexes/information_schema_search_index_columns.sql +++ b/models/base/google/search_indexes/information_schema_search_index_columns.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-index-columns -#} {# Required role/permissions: To see search index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) @@ -12,31 +13,5 @@ roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT index_catalog, index_schema, table_name, index_name, index_column_name, index_field_path - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEX_COLUMNS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -index_catalog, -index_schema, -table_name, -index_name, -index_column_name, -index_field_path +SELECT index_catalog, index_schema, table_name, index_name, index_column_name, index_field_path FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEX_COLUMNS` -{%- endif %} -) - -SELECT -index_catalog, -index_schema, -table_name, -index_name, -index_column_name, -index_field_path, -FROM -base diff --git a/models/base/google/search_indexes/information_schema_search_indexes.sql b/models/base/google/search_indexes/information_schema_search_indexes.sql index fa5a161..6b784b2 100644 --- a/models/base/google/search_indexes/information_schema_search_indexes.sql +++ b/models/base/google/search_indexes/information_schema_search_indexes.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-indexes -#} {# Required role/permissions: To see search index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) @@ -12,51 +13,5 @@ roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT index_catalog, index_schema, table_name, index_name, index_status, creation_time, last_modification_time, last_refresh_time, disable_time, disable_reason, ddl, coverage_percentage, unindexed_row_count, total_logical_bytes, total_storage_bytes, analyzer - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEXES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -index_catalog, -index_schema, -table_name, -index_name, -index_status, -creation_time, -last_modification_time, -last_refresh_time, -disable_time, -disable_reason, -ddl, -coverage_percentage, -unindexed_row_count, -total_logical_bytes, -total_storage_bytes, -analyzer +SELECT index_catalog, index_schema, table_name, index_name, index_status, creation_time, last_modification_time, last_refresh_time, disable_time, disable_reason, ddl, coverage_percentage, unindexed_row_count, total_logical_bytes, total_storage_bytes, analyzer FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEXES` -{%- endif %} -) - -SELECT -index_catalog, -index_schema, -table_name, -index_name, -index_status, -creation_time, -last_modification_time, -last_refresh_time, -disable_time, -disable_reason, -ddl, -coverage_percentage, -unindexed_row_count, -total_logical_bytes, -total_storage_bytes, -analyzer, -FROM -base diff --git a/models/base/google/sessions/information_schema_sessions_by_project.sql b/models/base/google/sessions/information_schema_sessions_by_project.sql index ba3bc11..c1b1fe7 100644 --- a/models/base/google/sessions/information_schema_sessions_by_project.sql +++ b/models/base/google/sessions/information_schema_sessions_by_project.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-sessions-by-project -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.SESSIONS_BY_PROJECT view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the project. @@ -8,35 +9,5 @@ BigQuery Admin For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT creation_time, expiration_time, is_active, last_modified_time, project_id, project_number, session_id, user_email - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_PROJECT` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -creation_time, -expiration_time, -is_active, -last_modified_time, -project_id, -project_number, -session_id, -user_email +SELECT creation_time, expiration_time, is_active, last_modified_time, project_id, project_number, session_id, user_email FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_PROJECT` -{%- endif %} -) - -SELECT -creation_time, -expiration_time, -is_active, -last_modified_time, -project_id, -project_number, -session_id, -user_email, -FROM -base diff --git a/models/base/google/sessions/information_schema_sessions_by_user.sql b/models/base/google/sessions/information_schema_sessions_by_user.sql index ffb4105..3fb67a0 100644 --- a/models/base/google/sessions/information_schema_sessions_by_user.sql +++ b/models/base/google/sessions/information_schema_sessions_by_user.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-sessions-by-user -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.SESSIONS_BY_USER view, you need the bigquery.jobs.list Identity and Access Management (IAM) permission for the project. @@ -8,35 +9,5 @@ BigQuery User For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT creation_time, expiration_time, is_active, last_modified_time, project_id, project_number, session_id, user_email - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_USER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -creation_time, -expiration_time, -is_active, -last_modified_time, -project_id, -project_number, -session_id, -user_email +SELECT creation_time, expiration_time, is_active, last_modified_time, project_id, project_number, session_id, user_email FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_USER` -{%- endif %} -) - -SELECT -creation_time, -expiration_time, -is_active, -last_modified_time, -project_id, -project_number, -session_id, -user_email, -FROM -base diff --git a/models/base/google/streaming/information_schema_streaming_timeline.sql b/models/base/google/streaming/information_schema_streaming_timeline.sql index 01075ae..5ef35b0 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline.sql +++ b/models/base/google/streaming/information_schema_streaming_timeline.sql @@ -1,36 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -error_code, -total_requests, -total_rows, -total_input_bytes +SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE` -{%- endif %} -) - -SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -error_code, -total_requests, -total_rows, -total_input_bytes, -FROM -base diff --git a/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql b/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql index dc3e352..74111b9 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql +++ b/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql @@ -1,36 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming-by-folder -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_FOLDER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -error_code, -total_requests, -total_rows, -total_input_bytes +SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_FOLDER` -{%- endif %} -) - -SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -error_code, -total_requests, -total_rows, -total_input_bytes, -FROM -base diff --git a/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql b/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql index 4534905..5d99202 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql +++ b/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql @@ -1,36 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming-by-organization -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_ORGANIZATION` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -error_code, -total_requests, -total_rows, -total_input_bytes +SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_ORGANIZATION` -{%- endif %} -) - -SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -error_code, -total_requests, -total_rows, -total_input_bytes, -FROM -base diff --git a/models/base/google/tables/information_schema_column_field_paths.sql b/models/base/google/tables/information_schema_column_field_paths.sql index 254a42c..d1f2f04 100644 --- a/models/base/google/tables/information_schema_column_field_paths.sql +++ b/models/base/google/tables/information_schema_column_field_paths.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-column-field-paths -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.COLUMN_FIELD_PATHS view, you need the following Identity and Access Management (IAM) permissions: @@ -12,37 +13,5 @@ roles/bigquery.metadataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT table_catalog, table_schema, table_name, column_name, field_path, data_type, description, collation_name, rounding_mode - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMN_FIELD_PATHS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -table_catalog, -table_schema, -table_name, -column_name, -field_path, -data_type, -description, -collation_name, -rounding_mode +SELECT table_catalog, table_schema, table_name, column_name, field_path, data_type, description, collation_name, rounding_mode FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMN_FIELD_PATHS` -{%- endif %} -) - -SELECT -table_catalog, -table_schema, -table_name, -column_name, -field_path, -data_type, -description, -collation_name, -rounding_mode, -FROM -base diff --git a/models/base/google/tables/information_schema_columns.sql b/models/base/google/tables/information_schema_columns.sql index 4660446..7c48ba8 100644 --- a/models/base/google/tables/information_schema_columns.sql +++ b/models/base/google/tables/information_schema_columns.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-columns -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.COLUMNS view, you need the following Identity and Access Management (IAM) permissions: @@ -12,55 +13,5 @@ roles/bigquery.metadataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT table_catalog, table_schema, table_name, column_name, ordinal_position, is_nullable, data_type, is_generated, generation_expression, is_stored, is_hidden, is_updatable, is_system_defined, is_partitioning_column, clustering_ordinal_position, collation_name, column_default, rounding_mode - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMNS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -table_catalog, -table_schema, -table_name, -column_name, -ordinal_position, -is_nullable, -data_type, -is_generated, -generation_expression, -is_stored, -is_hidden, -is_updatable, -is_system_defined, -is_partitioning_column, -clustering_ordinal_position, -collation_name, -column_default, -rounding_mode +SELECT table_catalog, table_schema, table_name, column_name, ordinal_position, is_nullable, data_type, is_generated, generation_expression, is_stored, is_hidden, is_updatable, is_system_defined, is_partitioning_column, clustering_ordinal_position, collation_name, column_default, rounding_mode FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMNS` -{%- endif %} -) - -SELECT -table_catalog, -table_schema, -table_name, -column_name, -ordinal_position, -is_nullable, -data_type, -is_generated, -generation_expression, -is_stored, -is_hidden, -is_updatable, -is_system_defined, -is_partitioning_column, -clustering_ordinal_position, -collation_name, -column_default, -rounding_mode, -FROM -base diff --git a/models/base/google/tables/information_schema_constraint_column_usage.sql b/models/base/google/tables/information_schema_constraint_column_usage.sql index 21a8d81..8a78978 100644 --- a/models/base/google/tables/information_schema_constraint_column_usage.sql +++ b/models/base/google/tables/information_schema_constraint_column_usage.sql @@ -1,24 +1,16 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-constraint-column-usage -#} + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-constraint-column-usage -#} {% set preflight_sql -%} - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT - CONCAT('`', CATALOG_NAME, '`.`', SCHEMA_NAME, '`') AS SCHEMA_NAME - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} - {%- else %} SELECT CONCAT('`', CATALOG_NAME, '`.`', SCHEMA_NAME, '`') AS SCHEMA_NAME FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` - {%- endif %} {%- endset %} {% set results = run_query(preflight_sql) %} {% set dataset_list = results | map(attribute='SCHEMA_NAME') | list %} {%- if dataset_list | length == 0 -%} - {{ log("No datasets found in the project list", info=True) }} + {{ log("No datasets found in the project list", info=False) }} {%- endif -%} WITH base AS ( @@ -27,27 +19,14 @@ LIMIT 0 {%- else %} {% for dataset in dataset_list -%} - SELECT -table_catalog, -table_schema, -table_name, -column_name, -constraint_catalog, -constraint_schema, -constraint_name + SELECT table_catalog, table_schema, table_name, column_name, constraint_catalog, constraint_schema, constraint_name FROM {{ dataset | trim }}.`INFORMATION_SCHEMA`.`PARTITIONS` {% if not loop.last %}UNION ALL{% endif %} {% endfor %} {%- endif -%} ) - -SELECT - table_catalog, -table_schema, -table_name, -column_name, -constraint_catalog, -constraint_schema, -constraint_name, + SELECT + table_catalog, table_schema, table_name, column_name, constraint_catalog, constraint_schema, constraint_name, FROM base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_key_column_usage.sql b/models/base/google/tables/information_schema_key_column_usage.sql index 5eecaca..8688ba0 100644 --- a/models/base/google/tables/information_schema_key_column_usage.sql +++ b/models/base/google/tables/information_schema_key_column_usage.sql @@ -1,24 +1,16 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-key-column-usage -#} + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-key-column-usage -#} {% set preflight_sql -%} - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT - CONCAT('`', CATALOG_NAME, '`.`', SCHEMA_NAME, '`') AS SCHEMA_NAME - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} - {%- else %} SELECT CONCAT('`', CATALOG_NAME, '`.`', SCHEMA_NAME, '`') AS SCHEMA_NAME FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` - {%- endif %} {%- endset %} {% set results = run_query(preflight_sql) %} {% set dataset_list = results | map(attribute='SCHEMA_NAME') | list %} {%- if dataset_list | length == 0 -%} - {{ log("No datasets found in the project list", info=True) }} + {{ log("No datasets found in the project list", info=False) }} {%- endif -%} WITH base AS ( @@ -27,31 +19,14 @@ LIMIT 0 {%- else %} {% for dataset in dataset_list -%} - SELECT -constraint_catalog, -constraint_schema, -constraint_name, -table_catalog, -table_schema, -table_name, -column_name, -ordinal_position, -position_in_unique_constraint + SELECT constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, column_name, ordinal_position, position_in_unique_constraint FROM {{ dataset | trim }}.`INFORMATION_SCHEMA`.`PARTITIONS` {% if not loop.last %}UNION ALL{% endif %} {% endfor %} {%- endif -%} ) - -SELECT - constraint_catalog, -constraint_schema, -constraint_name, -table_catalog, -table_schema, -table_name, -column_name, -ordinal_position, -position_in_unique_constraint, + SELECT + constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, column_name, ordinal_position, position_in_unique_constraint, FROM base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_partitions.sql b/models/base/google/tables/information_schema_partitions.sql index ded90ab..d02b032 100644 --- a/models/base/google/tables/information_schema_partitions.sql +++ b/models/base/google/tables/information_schema_partitions.sql @@ -1,4 +1,5 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-partitions -#} + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-partitions -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.PARTITIONS view, you need the following Identity and Access Management (IAM) permissions: bigquery.tables.get @@ -14,23 +15,14 @@ Access control with IAM. -#} {% set preflight_sql -%} - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT - CONCAT('`', CATALOG_NAME, '`.`', SCHEMA_NAME, '`') AS SCHEMA_NAME - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} - {%- else %} SELECT CONCAT('`', CATALOG_NAME, '`.`', SCHEMA_NAME, '`') AS SCHEMA_NAME FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` - {%- endif %} {%- endset %} {% set results = run_query(preflight_sql) %} {% set dataset_list = results | map(attribute='SCHEMA_NAME') | list %} {%- if dataset_list | length == 0 -%} - {{ log("No datasets found in the project list", info=True) }} + {{ log("No datasets found in the project list", info=False) }} {%- endif -%} WITH base AS ( @@ -39,29 +31,14 @@ Access control with IAM. -#} LIMIT 0 {%- else %} {% for dataset in dataset_list -%} - SELECT -table_catalog, -table_schema, -table_name, -partition_id, -total_rows, -total_logical_bytes, -last_modified_time, -storage_tier + SELECT table_catalog, table_schema, table_name, partition_id, total_rows, total_logical_bytes, last_modified_time, storage_tier FROM {{ dataset | trim }}.`INFORMATION_SCHEMA`.`PARTITIONS` {% if not loop.last %}UNION ALL{% endif %} {% endfor %} {%- endif -%} ) - -SELECT - table_catalog, -table_schema, -table_name, -partition_id, -total_rows, -total_logical_bytes, -last_modified_time, -storage_tier, + SELECT + table_catalog, table_schema, table_name, partition_id, total_rows, total_logical_bytes, last_modified_time, storage_tier, FROM base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_table_constraints.sql b/models/base/google/tables/information_schema_table_constraints.sql index 0d091d5..cf824bb 100644 --- a/models/base/google/tables/information_schema_table_constraints.sql +++ b/models/base/google/tables/information_schema_table_constraints.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-constraints -#} {# Required role/permissions: You need the following Identity and Access Management (IAM) permissions: @@ -15,39 +16,5 @@ excess permissions.For more information about IAM roles and permissions in BigQuery, see Predefined roles and permissions. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, constraint_type, is_deferrable, initially_deferred, enforced - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_CONSTRAINTS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -constraint_catalog, -constraint_schema, -constraint_name, -table_catalog, -table_schema, -table_name, -constraint_type, -is_deferrable, -initially_deferred, -enforced +SELECT constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, constraint_type, is_deferrable, initially_deferred, enforced FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_CONSTRAINTS` -{%- endif %} -) - -SELECT -constraint_catalog, -constraint_schema, -constraint_name, -table_catalog, -table_schema, -table_name, -constraint_type, -is_deferrable, -initially_deferred, -enforced, -FROM -base diff --git a/models/base/google/tables/information_schema_table_options.sql b/models/base/google/tables/information_schema_table_options.sql index 3257780..c7c2c4c 100644 --- a/models/base/google/tables/information_schema_table_options.sql +++ b/models/base/google/tables/information_schema_table_options.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-options -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_OPTIONS view, you need the following Identity and Access Management (IAM) permissions: @@ -13,31 +14,5 @@ roles/bigquery.metadataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT table_catalog, table_schema, table_name, option_name, option_type, option_value - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_OPTIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -table_catalog, -table_schema, -table_name, -option_name, -option_type, -option_value +SELECT table_catalog, table_schema, table_name, option_name, option_type, option_value FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_OPTIONS` -{%- endif %} -) - -SELECT -table_catalog, -table_schema, -table_name, -option_name, -option_type, -option_value, -FROM -base diff --git a/models/base/google/tables/information_schema_table_snapshots.sql b/models/base/google/tables/information_schema_table_snapshots.sql index f60c61c..829b21d 100644 --- a/models/base/google/tables/information_schema_table_snapshots.sql +++ b/models/base/google/tables/information_schema_table_snapshots.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-snapshots -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_SNAPSHOTS view, you need the bigquery.tables.list Identity and Access Management (IAM) permission for the dataset. @@ -5,33 +6,5 @@ The roles/bigquery.metadataViewer predefined role includes the required permission.For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT table_catalog, table_schema, table_name, base_table_catalog, base_table_schema, base_table_name, snapshot_time - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_SNAPSHOTS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -table_catalog, -table_schema, -table_name, -base_table_catalog, -base_table_schema, -base_table_name, -snapshot_time +SELECT table_catalog, table_schema, table_name, base_table_catalog, base_table_schema, base_table_name, snapshot_time FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_SNAPSHOTS` -{%- endif %} -) - -SELECT -table_catalog, -table_schema, -table_name, -base_table_catalog, -base_table_schema, -base_table_name, -snapshot_time, -FROM -base diff --git a/models/base/google/tables/information_schema_table_storage.sql b/models/base/google/tables/information_schema_table_storage.sql index abcb8bf..bb290cb 100644 --- a/models/base/google/tables/information_schema_table_storage.sql +++ b/models/base/google/tables/information_schema_table_storage.sql @@ -1,60 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT project_id, project_number, table_catalog, table_schema, table_name, creation_time, total_rows, total_partitions, total_logical_bytes, active_logical_bytes, long_term_logical_bytes, current_physical_bytes, total_physical_bytes, active_physical_bytes, long_term_physical_bytes, time_travel_physical_bytes, storage_last_modified_time, deleted, table_type, fail_safe_physical_bytes, last_metadata_index_refresh_time - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -project_id, -project_number, -table_catalog, -table_schema, -table_name, -creation_time, -total_rows, -total_partitions, -total_logical_bytes, -active_logical_bytes, -long_term_logical_bytes, -current_physical_bytes, -total_physical_bytes, -active_physical_bytes, -long_term_physical_bytes, -time_travel_physical_bytes, -storage_last_modified_time, -deleted, -table_type, -fail_safe_physical_bytes, -last_metadata_index_refresh_time +SELECT project_id, project_number, table_catalog, table_schema, table_name, creation_time, total_rows, total_partitions, total_logical_bytes, active_logical_bytes, long_term_logical_bytes, current_physical_bytes, total_physical_bytes, active_physical_bytes, long_term_physical_bytes, time_travel_physical_bytes, storage_last_modified_time, deleted, table_type, fail_safe_physical_bytes, last_metadata_index_refresh_time FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE` -{%- endif %} -) - -SELECT -project_id, -project_number, -table_catalog, -table_schema, -table_name, -creation_time, -total_rows, -total_partitions, -total_logical_bytes, -active_logical_bytes, -long_term_logical_bytes, -current_physical_bytes, -total_physical_bytes, -active_physical_bytes, -long_term_physical_bytes, -time_travel_physical_bytes, -storage_last_modified_time, -deleted, -table_type, -fail_safe_physical_bytes, -last_metadata_index_refresh_time, -FROM -base diff --git a/models/base/google/tables/information_schema_table_storage_by_organization.sql b/models/base/google/tables/information_schema_table_storage_by_organization.sql index 533b9d1..5f16ddf 100644 --- a/models/base/google/tables/information_schema_table_storage_by_organization.sql +++ b/models/base/google/tables/information_schema_table_storage_by_organization.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-by-organization -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_BY_ORGANIZATION view, you need the following Identity and Access Management (IAM) permissions for your organization: @@ -13,61 +14,5 @@ This schema view is only available to users with defined Google Cloud organizations.For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT project_id, project_number, table_catalog, table_schema, table_name, creation_time, total_rows, total_partitions, total_logical_bytes, active_logical_bytes, long_term_logical_bytes, current_physical_bytes, total_physical_bytes, active_physical_bytes, long_term_physical_bytes, time_travel_physical_bytes, storage_last_modified_time, deleted, table_type, fail_safe_physical_bytes, last_metadata_index_refresh_time - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_BY_ORGANIZATION` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -project_id, -project_number, -table_catalog, -table_schema, -table_name, -creation_time, -total_rows, -total_partitions, -total_logical_bytes, -active_logical_bytes, -long_term_logical_bytes, -current_physical_bytes, -total_physical_bytes, -active_physical_bytes, -long_term_physical_bytes, -time_travel_physical_bytes, -storage_last_modified_time, -deleted, -table_type, -fail_safe_physical_bytes, -last_metadata_index_refresh_time +SELECT project_id, project_number, table_catalog, table_schema, table_name, creation_time, total_rows, total_partitions, total_logical_bytes, active_logical_bytes, long_term_logical_bytes, current_physical_bytes, total_physical_bytes, active_physical_bytes, long_term_physical_bytes, time_travel_physical_bytes, storage_last_modified_time, deleted, table_type, fail_safe_physical_bytes, last_metadata_index_refresh_time FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_BY_ORGANIZATION` -{%- endif %} -) - -SELECT -project_id, -project_number, -table_catalog, -table_schema, -table_name, -creation_time, -total_rows, -total_partitions, -total_logical_bytes, -active_logical_bytes, -long_term_logical_bytes, -current_physical_bytes, -total_physical_bytes, -active_physical_bytes, -long_term_physical_bytes, -time_travel_physical_bytes, -storage_last_modified_time, -deleted, -table_type, -fail_safe_physical_bytes, -last_metadata_index_refresh_time, -FROM -base diff --git a/models/base/google/tables/information_schema_table_storage_usage_timeline.sql b/models/base/google/tables/information_schema_table_storage_usage_timeline.sql index 7a5d6f4..ef1d427 100644 --- a/models/base/google/tables/information_schema_table_storage_usage_timeline.sql +++ b/models/base/google/tables/information_schema_table_storage_usage_timeline.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-usage -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_USAGE_TIMELINE view, you need the following Identity and Access Management (IAM) permissions: @@ -12,41 +13,5 @@ roles/bigquery.admin For queries with a region qualifier, you must have permissions for the project.For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT project_id, table_catalog, project_number, table_schema, table_name, billable_total_logical_usage, billable_active_logical_usage, billable_long_term_logical_usage, billable_total_physical_usage, billable_active_physical_usage, billable_long_term_physical_usage - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -project_id, -table_catalog, -project_number, -table_schema, -table_name, -billable_total_logical_usage, -billable_active_logical_usage, -billable_long_term_logical_usage, -billable_total_physical_usage, -billable_active_physical_usage, -billable_long_term_physical_usage +SELECT project_id, table_catalog, project_number, table_schema, table_name, billable_total_logical_usage, billable_active_logical_usage, billable_long_term_logical_usage, billable_total_physical_usage, billable_active_physical_usage, billable_long_term_physical_usage FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE` -{%- endif %} -) - -SELECT -project_id, -table_catalog, -project_number, -table_schema, -table_name, -billable_total_logical_usage, -billable_active_logical_usage, -billable_long_term_logical_usage, -billable_total_physical_usage, -billable_active_physical_usage, -billable_long_term_physical_usage, -FROM -base diff --git a/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql b/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql index a2185e2..43569a5 100644 --- a/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql +++ b/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-usage-by-organization -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_USAGE_TIMELINE_BY_ORGANIZATION view, you need the following Identity and Access Management (IAM) permissions for your organization: @@ -13,41 +14,5 @@ This schema view is only available to users with defined Google Cloud organizations.For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT project_id, table_catalog, project_number, table_schema, table_name, billable_total_logical_usage, billable_active_logical_usage, billable_long_term_logical_usage, billable_total_physical_usage, billable_active_physical_usage, billable_long_term_physical_usage - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE_BY_ORGANIZATION` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -project_id, -table_catalog, -project_number, -table_schema, -table_name, -billable_total_logical_usage, -billable_active_logical_usage, -billable_long_term_logical_usage, -billable_total_physical_usage, -billable_active_physical_usage, -billable_long_term_physical_usage +SELECT project_id, table_catalog, project_number, table_schema, table_name, billable_total_logical_usage, billable_active_logical_usage, billable_long_term_logical_usage, billable_total_physical_usage, billable_active_physical_usage, billable_long_term_physical_usage FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE_BY_ORGANIZATION` -{%- endif %} -) - -SELECT -project_id, -table_catalog, -project_number, -table_schema, -table_name, -billable_total_logical_usage, -billable_active_logical_usage, -billable_long_term_logical_usage, -billable_total_physical_usage, -billable_active_physical_usage, -billable_long_term_physical_usage, -FROM -base diff --git a/models/base/google/tables/information_schema_tables.sql b/models/base/google/tables/information_schema_tables.sql index e9e3633..c67a459 100644 --- a/models/base/google/tables/information_schema_tables.sql +++ b/models/base/google/tables/information_schema_tables.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-tables -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLES view, you need the following Identity and Access Management (IAM) permissions: @@ -13,59 +14,5 @@ roles/bigquery.metadataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT table_catalog, table_schema, table_name, table_type, is_insertable_into, is_typed, is_change_history_enabled, creation_time, base_table_catalog, base_table_schema, base_table_name, snapshot_time_ms, replica_source_catalog, replica_source_schema, replica_source_name, replication_status, replication_error, ddl, default_collation_name, upsert_stream_apply_watermark - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -table_catalog, -table_schema, -table_name, -table_type, -is_insertable_into, -is_typed, -is_change_history_enabled, -creation_time, -base_table_catalog, -base_table_schema, -base_table_name, -snapshot_time_ms, -replica_source_catalog, -replica_source_schema, -replica_source_name, -replication_status, -replication_error, -ddl, -default_collation_name, -upsert_stream_apply_watermark +SELECT table_catalog, table_schema, table_name, table_type, is_insertable_into, is_typed, is_change_history_enabled, creation_time, base_table_catalog, base_table_schema, base_table_name, snapshot_time_ms, replica_source_catalog, replica_source_schema, replica_source_name, replication_status, replication_error, ddl, default_collation_name, upsert_stream_apply_watermark FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLES` -{%- endif %} -) - -SELECT -table_catalog, -table_schema, -table_name, -table_type, -is_insertable_into, -is_typed, -is_change_history_enabled, -creation_time, -base_table_catalog, -base_table_schema, -base_table_name, -snapshot_time_ms, -replica_source_catalog, -replica_source_schema, -replica_source_name, -replication_status, -replication_error, -ddl, -default_collation_name, -upsert_stream_apply_watermark, -FROM -base diff --git a/models/base/google/vector_indexes/information_schema_vector_index_columns.sql b/models/base/google/vector_indexes/information_schema_vector_index_columns.sql index cd98a28..a893fff 100644 --- a/models/base/google/vector_indexes/information_schema_vector_index_columns.sql +++ b/models/base/google/vector_indexes/information_schema_vector_index_columns.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-index-columns -#} {# Required role/permissions: To see vector index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) @@ -12,31 +13,5 @@ roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT index_catalog, index_schema, table_name, index_name, index_column_name, index_field_path - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_COLUMNS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -index_catalog, -index_schema, -table_name, -index_name, -index_column_name, -index_field_path +SELECT index_catalog, index_schema, table_name, index_name, index_column_name, index_field_path FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_COLUMNS` -{%- endif %} -) - -SELECT -index_catalog, -index_schema, -table_name, -index_name, -index_column_name, -index_field_path, -FROM -base diff --git a/models/base/google/vector_indexes/information_schema_vector_index_options.sql b/models/base/google/vector_indexes/information_schema_vector_index_options.sql index 3838694..63bb469 100644 --- a/models/base/google/vector_indexes/information_schema_vector_index_options.sql +++ b/models/base/google/vector_indexes/information_schema_vector_index_options.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-index-options -#} {# Required role/permissions: To see vector index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) @@ -12,33 +13,5 @@ roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT index_catalog, index_schema, table_name, index_name, option_name, option_type, option_value - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_OPTIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -index_catalog, -index_schema, -table_name, -index_name, -option_name, -option_type, -option_value +SELECT index_catalog, index_schema, table_name, index_name, option_name, option_type, option_value FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_OPTIONS` -{%- endif %} -) - -SELECT -index_catalog, -index_schema, -table_name, -index_name, -option_name, -option_type, -option_value, -FROM -base diff --git a/models/base/google/vector_indexes/information_schema_vector_indexes.sql b/models/base/google/vector_indexes/information_schema_vector_indexes.sql index 9f1c778..5bc180f 100644 --- a/models/base/google/vector_indexes/information_schema_vector_indexes.sql +++ b/models/base/google/vector_indexes/information_schema_vector_indexes.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-indexes -#} {# Required role/permissions: To see vector index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) @@ -12,49 +13,5 @@ roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT index_catalog, index_schema, table_name, index_name, index_status, creation_time, last_modification_time, last_refresh_time, disable_time, disable_reason, ddl, coverage_percentage, unindexed_row_count, total_logical_bytes, total_storage_bytes - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEXES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -index_catalog, -index_schema, -table_name, -index_name, -index_status, -creation_time, -last_modification_time, -last_refresh_time, -disable_time, -disable_reason, -ddl, -coverage_percentage, -unindexed_row_count, -total_logical_bytes, -total_storage_bytes +SELECT index_catalog, index_schema, table_name, index_name, index_status, creation_time, last_modification_time, last_refresh_time, disable_time, disable_reason, ddl, coverage_percentage, unindexed_row_count, total_logical_bytes, total_storage_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEXES` -{%- endif %} -) - -SELECT -index_catalog, -index_schema, -table_name, -index_name, -index_status, -creation_time, -last_modification_time, -last_refresh_time, -disable_time, -disable_reason, -ddl, -coverage_percentage, -unindexed_row_count, -total_logical_bytes, -total_storage_bytes, -FROM -base diff --git a/models/base/google/views/information_schema_materialized_views.sql b/models/base/google/views/information_schema_materialized_views.sql index 5e7290c..8686c00 100644 --- a/models/base/google/views/information_schema_materialized_views.sql +++ b/models/base/google/views/information_schema_materialized_views.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-materialized-views -#} {# Required role/permissions: @@ -40,31 +41,5 @@ The following permissions are required to query the INFORMATION_SCHEMA.MATERIALI other predefined roles. Access control with IAM -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT table_catalog, table_schema, table_name, last_refresh_time, refresh_watermark, last_refresh_status - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`MATERIALIZED_VIEWS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -table_catalog, -table_schema, -table_name, -last_refresh_time, -refresh_watermark, -last_refresh_status +SELECT table_catalog, table_schema, table_name, last_refresh_time, refresh_watermark, last_refresh_status FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`MATERIALIZED_VIEWS` -{%- endif %} -) - -SELECT -table_catalog, -table_schema, -table_name, -last_refresh_time, -refresh_watermark, -last_refresh_status, -FROM -base diff --git a/models/base/google/views/information_schema_views.sql b/models/base/google/views/information_schema_views.sql index e26c8ed..544bac2 100644 --- a/models/base/google/views/information_schema_views.sql +++ b/models/base/google/views/information_schema_views.sql @@ -1,3 +1,4 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-views -#} {# Required role/permissions: To get view metadata, you need the following Identity and Access Management (IAM) permissions: @@ -12,31 +13,5 @@ roles/bigquery.dataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT table_catalog, table_schema, table_name, view_definition, check_option, use_standard_sql - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VIEWS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -table_catalog, -table_schema, -table_name, -view_definition, -check_option, -use_standard_sql +SELECT table_catalog, table_schema, table_name, view_definition, check_option, use_standard_sql FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VIEWS` -{%- endif %} -) - -SELECT -table_catalog, -table_schema, -table_name, -view_definition, -check_option, -use_standard_sql, -FROM -base diff --git a/models/base/google/write_api/information_schema_write_api_timeline.sql b/models/base/google/write_api/information_schema_write_api_timeline.sql index b4fa0a7..fd27a52 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline.sql +++ b/models/base/google/write_api/information_schema_write_api_timeline.sql @@ -1,38 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -stream_type, -error_code, -total_requests, -total_rows, -total_input_bytes +SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE` -{%- endif %} -) - -SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -stream_type, -error_code, -total_requests, -total_rows, -total_input_bytes, -FROM -base diff --git a/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql b/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql index a1eb7cb..4ac41b8 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql +++ b/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql @@ -1,38 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api-by-folder -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_FOLDER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -stream_type, -error_code, -total_requests, -total_rows, -total_input_bytes +SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_FOLDER` -{%- endif %} -) - -SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -stream_type, -error_code, -total_requests, -total_rows, -total_input_bytes, -FROM -base diff --git a/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql b/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql index f7fd0c3..7d81704 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql +++ b/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql @@ -1,38 +1,5 @@ +{{ config(materialization='project_by_project_table') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api-by-organization -#} -WITH base AS ( - {% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes - FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_ORGANIZATION` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -stream_type, -error_code, -total_requests, -total_rows, -total_input_bytes +SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_ORGANIZATION` -{%- endif %} -) - -SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -stream_type, -error_code, -total_requests, -total_rows, -total_input_bytes, -FROM -base From 595a7b8c10aaa3f186b06ba2acd6e5c5e67097a7 Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Sat, 11 Jan 2025 14:58:44 +0100 Subject: [PATCH 02/12] clean the table if required and use partition key if defined --- macros/project_by_project_table.sql | 20 +++++++++++++++++++ .../google/jobs/information_schema_jobs.sql | 4 ++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/macros/project_by_project_table.sql b/macros/project_by_project_table.sql index eaf1115..131f95c 100644 --- a/macros/project_by_project_table.sql +++ b/macros/project_by_project_table.sql @@ -5,6 +5,7 @@ {% set existing_relation = load_relation(this) %} {% set tmp_relation = make_temp_relation(this) %} {% set projects = project_list() %} + {% set partition_by = config.get('partition_by') %} {{ run_hooks(pre_hooks) }} @@ -12,12 +13,31 @@ {% if existing_relation is none %} {% set build_sql = create_table_as(False, target_relation, sql) %} {% do run_query(build_sql) %} + {% else %} + {% if partition_by is not none %} + -- Get the maximum partition value + {% set max_partition_sql %} + SELECT MAX({{ partition_by }}) as max_partition + FROM {{ target_relation }} + {% endset %} + {% set max_partition_result = run_query(max_partition_sql) %} + {% set max_partition_value = max_partition_result['data'][0]['max_partition'] %} + {% else %} + -- Truncate the table if partition_by is not defined + {% set truncate_sql %} + TRUNCATE TABLE {{ target_relation }} + {% endset %} + {% do run_query(truncate_sql) %} + {% endif %} {% endif %} -- If we have projects, process them one by one {% if projects|length > 0 %} {% for project in projects %} {% set project_sql = sql | replace('`region-', '`' ~ project | trim ~ '`.`region-') %} + {% if existing_relation is not none and partition_by is not none %} + {% set project_sql = project_sql + ' WHERE ' ~ partition_by ~ ' >= "' ~ max_partition_value ~ '"' %} + {% endif %} {% set insert_sql %} INSERT INTO {{ target_relation }} {{ project_sql }} diff --git a/models/base/google/jobs/information_schema_jobs.sql b/models/base/google/jobs/information_schema_jobs.sql index d812669..c49aa06 100644 --- a/models/base/google/jobs/information_schema_jobs.sql +++ b/models/base/google/jobs/information_schema_jobs.sql @@ -1,6 +1,6 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization='project_by_project_table', partition_by='creation_time') }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs -#} -{# Required role/permissions: +{# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS view, From 949622b9df28f63bdf9903df8ccca1eb99fe57e9 Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Sat, 11 Jan 2025 15:50:37 +0100 Subject: [PATCH 03/12] add input partitioning --- macros/project_by_project_table.sql | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/macros/project_by_project_table.sql b/macros/project_by_project_table.sql index 131f95c..d215a0d 100644 --- a/macros/project_by_project_table.sql +++ b/macros/project_by_project_table.sql @@ -5,13 +5,17 @@ {% set existing_relation = load_relation(this) %} {% set tmp_relation = make_temp_relation(this) %} {% set projects = project_list() %} - {% set partition_by = config.get('partition_by') %} + {% set partition_by = config.get('input_partition_by') %} {{ run_hooks(pre_hooks) }} -- Create the table if it doesn't exist {% if existing_relation is none %} - {% set build_sql = create_table_as(False, target_relation, sql) %} + {% if partition_by is not none %} + {% set build_sql = create_table_as(False, target_relation, sql, partition_by=partition_by) %} + {% else %} + {% set build_sql = create_table_as(False, target_relation, sql) %} + {% endif %} {% do run_query(build_sql) %} {% else %} {% if partition_by is not none %} From 97fdb1c22f0cb44c11b095e68507dd2366d3b709 Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Sat, 11 Jan 2025 21:18:20 +0100 Subject: [PATCH 04/12] adapt for call statement --- ...dbt_bigquery_monitoring_materalization.sql | 9 ++ macros/project_by_project_table.sql | 103 ++++++++++++------ .../information_schema_object_privileges.sql | 2 +- .../information_schema_bi_capacities.sql | 2 +- ...information_schema_bi_capacity_changes.sql | 2 +- .../datasets/information_schema_schemata.sql | 2 +- .../information_schema_schemata_links.sql | 2 +- .../information_schema_schemata_options.sql | 2 +- .../information_schema_schemata_replicas.sql | 4 +- ...nformation_schema_shared_dataset_usage.sql | 2 +- .../google/jobs/information_schema_jobs.sql | 2 +- .../information_schema_jobs_by_folder.sql | 4 +- ...nformation_schema_jobs_by_organization.sql | 4 +- .../information_schema_jobs_by_project.sql | 4 +- .../jobs/information_schema_jobs_by_user.sql | 4 +- .../information_schema_jobs_timeline.sql | 2 +- ...rmation_schema_jobs_timeline_by_folder.sql | 2 +- ...n_schema_jobs_timeline_by_organization.sql | 2 +- ...formation_schema_jobs_timeline_by_user.sql | 2 +- .../information_schema_insights.sql | 2 +- .../information_schema_recommendations.sql | 2 +- ...schema_recommendations_by_organization.sql | 2 +- .../information_schema_assignment_changes.sql | 2 +- .../information_schema_assignments.sql | 2 +- ...ion_schema_capacity_commitment_changes.sql | 2 +- ...nformation_schema_capacity_commitments.sql | 2 +- ...information_schema_reservation_changes.sql | 2 +- .../information_schema_reservations.sql | 2 +- ...formation_schema_reservations_timeline.sql | 2 +- .../information_schema_parameters.sql | 2 +- .../information_schema_routine_options.sql | 2 +- .../routines/information_schema_routines.sql | 2 +- ...nformation_schema_search_index_columns.sql | 2 +- .../information_schema_search_indexes.sql | 2 +- ...information_schema_sessions_by_project.sql | 2 +- .../information_schema_sessions_by_user.sql | 2 +- .../information_schema_streaming_timeline.sql | 2 +- ...on_schema_streaming_timeline_by_folder.sql | 2 +- ...ema_streaming_timeline_by_organization.sql | 2 +- .../information_schema_column_field_paths.sql | 2 +- .../tables/information_schema_columns.sql | 2 +- .../information_schema_table_constraints.sql | 2 +- .../information_schema_table_options.sql | 2 +- .../information_schema_table_snapshots.sql | 2 +- .../information_schema_table_storage.sql | 2 +- ...n_schema_table_storage_by_organization.sql | 2 +- ...on_schema_table_storage_usage_timeline.sql | 2 +- ...storage_usage_timeline_by_organization.sql | 2 +- .../tables/information_schema_tables.sql | 2 +- ...nformation_schema_vector_index_columns.sql | 2 +- ...nformation_schema_vector_index_options.sql | 2 +- .../information_schema_vector_indexes.sql | 2 +- .../information_schema_materialized_views.sql | 6 +- .../google/views/information_schema_views.sql | 2 +- .../information_schema_write_api_timeline.sql | 2 +- ...on_schema_write_api_timeline_by_folder.sql | 2 +- ...ema_write_api_timeline_by_organization.sql | 2 +- 57 files changed, 138 insertions(+), 98 deletions(-) create mode 100644 macros/dbt_bigquery_monitoring_materalization.sql diff --git a/macros/dbt_bigquery_monitoring_materalization.sql b/macros/dbt_bigquery_monitoring_materalization.sql new file mode 100644 index 0000000..eca8d8e --- /dev/null +++ b/macros/dbt_bigquery_monitoring_materalization.sql @@ -0,0 +1,9 @@ +{% macro dbt_bigquery_monitoring_materialization() %} + {% set projects = project_list() %} + {% if projects|length == 0 %} + {% set materialization = 'ephemeral' %} + {% else %} + {% set materialization = 'project_by_project_table' %} + {% endif %} + {{ return(materialization) }} +{% endmacro %} diff --git a/macros/project_by_project_table.sql b/macros/project_by_project_table.sql index d215a0d..5b40ace 100644 --- a/macros/project_by_project_table.sql +++ b/macros/project_by_project_table.sql @@ -1,58 +1,89 @@ -{% macro project_by_project_table() %} - {%- materialization 'project_by_project_table', adapter='bigquery' -%} +{%- materialization project_by_project_table, adapter='bigquery' -%} - {% set target_relation = this %} - {% set existing_relation = load_relation(this) %} - {% set tmp_relation = make_temp_relation(this) %} - {% set projects = project_list() %} - {% set partition_by = config.get('input_partition_by') %} +{% set target_relation = this %} +{% set existing_relation = load_relation(this) %} +{% set tmp_relation = make_temp_relation(this) %} +{% set projects = project_list() %} +{%- set raw_partition_by = config.get('partition_by', none) -%} +{%- set partition_config = adapter.parse_partition_by(raw_partition_by) -%} +{{ log("project_by_project_table with partition_by : " ~ partition_by, info=True) }} - {{ run_hooks(pre_hooks) }} +{{ run_hooks(pre_hooks) }} - -- Create the table if it doesn't exist - {% if existing_relation is none %} - {% if partition_by is not none %} - {% set build_sql = create_table_as(False, target_relation, sql, partition_by=partition_by) %} +-- Create the table if it doesn't exist +{% if existing_relation is none %} + {% call statement('main') -%} + {% if partition_config is not none %} + {{ log("Creating table with partition_by : " ~ partition_by, info=True) }} + {% set build_sql = create_table_as(False, target_relation, sql) %} {% else %} + {{ log("Creating table without partition_by", info=True) }} {% set build_sql = create_table_as(False, target_relation, sql) %} {% endif %} - {% do run_query(build_sql) %} - {% else %} - {% if partition_by is not none %} + {{ log("Running build_sql : " ~ build_sql, info=True) }} + {{ build_sql }} + {%- endcall %} +{% else %} + {% call statement('main') -%} + SELECT 1 + {%- endcall %} + {{ log("Table already exists", info=True) }} + {% if partition_config is not none %} -- Get the maximum partition value {% set max_partition_sql %} - SELECT MAX({{ partition_by }}) as max_partition + SELECT MAX({{ partition_config.field }}) as max_partition FROM {{ target_relation }} {% endset %} - {% set max_partition_result = run_query(max_partition_sql) %} - {% set max_partition_value = max_partition_result['data'][0]['max_partition'] %} + {{ log("Running max_partition_sql : " ~ max_partition_sql, info=True) }} {% else %} -- Truncate the table if partition_by is not defined {% set truncate_sql %} TRUNCATE TABLE {{ target_relation }} {% endset %} + {{ log("Running truncate_sql : " ~ truncate_sql, info=True) }} + {{ truncate_sql }} {% do run_query(truncate_sql) %} {% endif %} + {{ log("Check partition_config", info=True) }} + {% if partition_config is not none %} + {{ log("Getting max_partition_value", info=True) }} + {% set max_partition_result = run_query(max_partition_sql) %} + {{ log("max_partition_result : " ~ max_partition_result, info=True) }} + {% if max_partition_result['data']|length > 0 %} + {% set max_partition_value = max_partition_result['data'][0]['max_partition'] %} + {{ log("max_partition_value : " ~ max_partition_value, info=True) }} + {% else %} + {% set max_partition_value = none %} + {{ log("No max_partition_value found", info=True) }} + {% endif %} {% endif %} +{% endif %} - -- If we have projects, process them one by one - {% if projects|length > 0 %} - {% for project in projects %} - {% set project_sql = sql | replace('`region-', '`' ~ project | trim ~ '`.`region-') %} - {% if existing_relation is not none and partition_by is not none %} - {% set project_sql = project_sql + ' WHERE ' ~ partition_by ~ ' >= "' ~ max_partition_value ~ '"' %} - {% endif %} - {% set insert_sql %} - INSERT INTO {{ target_relation }} - {{ project_sql }} - {% endset %} - {% do run_query(insert_sql) %} - {% endfor %} - {% endif %} +-- If we have projects, process them one by one +{% if projects|length > 0 %} + {% set all_insert_sql = [] %} + {% for project in projects %} + {{ log("Processing project : " ~ project, info=True) }} + {% set project_sql = sql | replace('`region-', '`' ~ project | trim ~ '`.`region-') %} + {% if existing_relation is not none and partition_config is not none and max_partition_value is not none %} + {% set project_sql = project_sql + ' WHERE ' ~ partition_config.field ~ ' >= "' ~ max_partition_value ~ '"' %} + {% endif %} + {% set insert_sql %} + INSERT INTO {{ target_relation }} + {{ project_sql }} + {% endset %} + {{ log("Running insert_sql : " ~ insert_sql, info=True) }} + {% do all_insert_sql.append(insert_sql) %} + {% endfor %} + {{ log(" all_insert_sql : " ~ all_insert_sql, info=True) }} + {% call statement('insert') -%} + {{ all_insert_sql | join(';\n') }} + {%- endcall %} +{% endif %} - {{ run_hooks(post_hooks) }} +{{ log("Materialization complete", info=True) }} +{{ run_hooks(post_hooks) }} - {{ return({'relations': [target_relation]}) }} +{{ return({'relations': [target_relation]}) }} - {%- endmaterialization -%} -{% endmacro %} +{%- endmaterialization -%} diff --git a/models/base/google/access_control/information_schema_object_privileges.sql b/models/base/google/access_control/information_schema_object_privileges.sql index 3dd14c9..3859130 100644 --- a/models/base/google/access_control/information_schema_object_privileges.sql +++ b/models/base/google/access_control/information_schema_object_privileges.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-object-privileges -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.OBJECT_PRIVILEGES view, you need following Identity and Access Management (IAM) permissions: diff --git a/models/base/google/bi_engine/information_schema_bi_capacities.sql b/models/base/google/bi_engine/information_schema_bi_capacities.sql index 8dc3539..8ea7983 100644 --- a/models/base/google/bi_engine/information_schema_bi_capacities.sql +++ b/models/base/google/bi_engine/information_schema_bi_capacities.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-bi-capacities -#} SELECT project_id, project_number, bi_capacity_name, size, preferred_tables diff --git a/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql b/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql index a112536..7c8711b 100644 --- a/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql +++ b/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-bi-capacity-changes -#} SELECT change_timestamp, project_id, project_number, bi_capacity_name, size, user_email, preferred_tables diff --git a/models/base/google/datasets/information_schema_schemata.sql b/models/base/google/datasets/information_schema_schemata.sql index 13545ee..b09389c 100644 --- a/models/base/google/datasets/information_schema_schemata.sql +++ b/models/base/google/datasets/information_schema_schemata.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata -#} SELECT catalog_name, schema_name, schema_owner, creation_time, last_modified_time, location, ddl, default_collation_name diff --git a/models/base/google/datasets/information_schema_schemata_links.sql b/models/base/google/datasets/information_schema_schemata_links.sql index adc5ee0..83cc115 100644 --- a/models/base/google/datasets/information_schema_schemata_links.sql +++ b/models/base/google/datasets/information_schema_schemata_links.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata-links -#} SELECT catalog_name, schema_name, linked_schema_catalog_number, linked_schema_catalog_name, linked_schema_name, linked_schema_creation_time, linked_schema_org_display_name diff --git a/models/base/google/datasets/information_schema_schemata_options.sql b/models/base/google/datasets/information_schema_schemata_options.sql index 75922aa..7dc3958 100644 --- a/models/base/google/datasets/information_schema_schemata_options.sql +++ b/models/base/google/datasets/information_schema_schemata_options.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata-options -#} SELECT catalog_name, schema_name, option_name, option_type, option_value diff --git a/models/base/google/datasets/information_schema_schemata_replicas.sql b/models/base/google/datasets/information_schema_schemata_replicas.sql index fc8eb66..6d36202 100644 --- a/models/base/google/datasets/information_schema_schemata_replicas.sql +++ b/models/base/google/datasets/information_schema_schemata_replicas.sql @@ -1,6 +1,6 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-schemata-replicas -#} -{# Required role/permissions: +{# Required role/permissions: To get the permissions that you need to query the INFORMATION_SCHEMA.SCHEMATA_REPLICAS view, diff --git a/models/base/google/datasets/information_schema_shared_dataset_usage.sql b/models/base/google/datasets/information_schema_shared_dataset_usage.sql index d616b33..4af9e52 100644 --- a/models/base/google/datasets/information_schema_shared_dataset_usage.sql +++ b/models/base/google/datasets/information_schema_shared_dataset_usage.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-shared-dataset-usage -#} SELECT project_id, dataset_id, table_id, data_exchange_id, listing_id, job_start_time, job_end_time, job_id, job_project_number, job_location, linked_project_number, linked_dataset_id, subscriber_org_number, subscriber_org_display_name, num_rows_processed, total_bytes_processed diff --git a/models/base/google/jobs/information_schema_jobs.sql b/models/base/google/jobs/information_schema_jobs.sql index c49aa06..c79f852 100644 --- a/models/base/google/jobs/information_schema_jobs.sql +++ b/models/base/google/jobs/information_schema_jobs.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table', partition_by='creation_time') }} +{{ config(materialized=dbt_bigquery_monitoring_materialization(), partition_by={'field': 'creation_time', 'data_type': 'timestamp', 'granularity': 'hour'}) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs -#} {# Required role/permissions: diff --git a/models/base/google/jobs/information_schema_jobs_by_folder.sql b/models/base/google/jobs/information_schema_jobs_by_folder.sql index 2480acd..a690dbd 100644 --- a/models/base/google/jobs/information_schema_jobs_by_folder.sql +++ b/models/base/google/jobs/information_schema_jobs_by_folder.sql @@ -1,6 +1,6 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-folder -#} -{# Required role/permissions: +{# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS_BY_FOLDER view, diff --git a/models/base/google/jobs/information_schema_jobs_by_organization.sql b/models/base/google/jobs/information_schema_jobs_by_organization.sql index b1aa626..98229b3 100644 --- a/models/base/google/jobs/information_schema_jobs_by_organization.sql +++ b/models/base/google/jobs/information_schema_jobs_by_organization.sql @@ -1,6 +1,6 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-organization -#} -{# Required role/permissions: +{# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS_BY_ORGANIZATION view, diff --git a/models/base/google/jobs/information_schema_jobs_by_project.sql b/models/base/google/jobs/information_schema_jobs_by_project.sql index 667a3f2..333657f 100644 --- a/models/base/google/jobs/information_schema_jobs_by_project.sql +++ b/models/base/google/jobs/information_schema_jobs_by_project.sql @@ -1,6 +1,6 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs -#} -{# Required role/permissions: +{# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS view, diff --git a/models/base/google/jobs/information_schema_jobs_by_user.sql b/models/base/google/jobs/information_schema_jobs_by_user.sql index 0ed7e69..ed40d5e 100644 --- a/models/base/google/jobs/information_schema_jobs_by_user.sql +++ b/models/base/google/jobs/information_schema_jobs_by_user.sql @@ -1,6 +1,6 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-user -#} -{# Required role/permissions: +{# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS_BY_USER view, diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql index 7314c21..88279dd 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the project. diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql index 8f7bf6d..003a5c6 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-folder -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_FOLDER view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the parent diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql index 255113c..6054980 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-organization -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_ORGANIZATION view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the organization. diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql index 34f9426..1cce2fa 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-user -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_USER view, you need the bigquery.jobs.list Identity and Access Management (IAM) permission for the project. diff --git a/models/base/google/recommendations_and_insights/information_schema_insights.sql b/models/base/google/recommendations_and_insights/information_schema_insights.sql index 5f30e24..ab0e438 100644 --- a/models/base/google/recommendations_and_insights/information_schema_insights.sql +++ b/models/base/google/recommendations_and_insights/information_schema_insights.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-insights -#} SELECT insight_id, insight_type, subtype, project_id, project_number, description, last_updated_time, category, target_resources, state, severity, associated_recommendation_ids, additional_details diff --git a/models/base/google/recommendations_and_insights/information_schema_recommendations.sql b/models/base/google/recommendations_and_insights/information_schema_recommendations.sql index 3be4692..2653347 100644 --- a/models/base/google/recommendations_and_insights/information_schema_recommendations.sql +++ b/models/base/google/recommendations_and_insights/information_schema_recommendations.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-recommendations -#} SELECT recommendation_id, recommender, subtype, project_id, project_number, description, last_updated_time, target_resources, state, primary_impact, priority, associated_insight_ids, additional_details diff --git a/models/base/google/recommendations_and_insights/information_schema_recommendations_by_organization.sql b/models/base/google/recommendations_and_insights/information_schema_recommendations_by_organization.sql index 2fb2683..9d80028 100644 --- a/models/base/google/recommendations_and_insights/information_schema_recommendations_by_organization.sql +++ b/models/base/google/recommendations_and_insights/information_schema_recommendations_by_organization.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-recommendations-by-org -#} {# Required role/permissions: To view recommendations with the INFORMATION_SCHEMA.RECOMMENDATIONS_BY_ORGANIZATION view, you must have the diff --git a/models/base/google/reservations/information_schema_assignment_changes.sql b/models/base/google/reservations/information_schema_assignment_changes.sql index 742ae42..ac2e371 100644 --- a/models/base/google/reservations/information_schema_assignment_changes.sql +++ b/models/base/google/reservations/information_schema_assignment_changes.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-assignments-changes -#} SELECT change_timestamp, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type, action, user_email, state diff --git a/models/base/google/reservations/information_schema_assignments.sql b/models/base/google/reservations/information_schema_assignments.sql index 7e65b35..458fb4a 100644 --- a/models/base/google/reservations/information_schema_assignments.sql +++ b/models/base/google/reservations/information_schema_assignments.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-assignments -#} SELECT ddl, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type diff --git a/models/base/google/reservations/information_schema_capacity_commitment_changes.sql b/models/base/google/reservations/information_schema_capacity_commitment_changes.sql index b867aef..885e529 100644 --- a/models/base/google/reservations/information_schema_capacity_commitment_changes.sql +++ b/models/base/google/reservations/information_schema_capacity_commitment_changes.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-capacity-commitment-changes -#} SELECT change_timestamp, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, action, user_email, commitment_start_time, commitment_end_time, failure_status, renewal_plan, edition, is_flat_rate diff --git a/models/base/google/reservations/information_schema_capacity_commitments.sql b/models/base/google/reservations/information_schema_capacity_commitments.sql index c50a12f..7e9f864 100644 --- a/models/base/google/reservations/information_schema_capacity_commitments.sql +++ b/models/base/google/reservations/information_schema_capacity_commitments.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-capacity-commitments -#} SELECT ddl, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, edition, is_flat_rate, renewal_plan diff --git a/models/base/google/reservations/information_schema_reservation_changes.sql b/models/base/google/reservations/information_schema_reservation_changes.sql index 77e6088..5734d0f 100644 --- a/models/base/google/reservations/information_schema_reservation_changes.sql +++ b/models/base/google/reservations/information_schema_reservation_changes.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservation-changes -#} SELECT change_timestamp, project_id, project_number, reservation_name, ignore_idle_slots, action, slot_capacity, user_email, target_job_concurrency, autoscale, edition diff --git a/models/base/google/reservations/information_schema_reservations.sql b/models/base/google/reservations/information_schema_reservations.sql index 235cdfd..c7b1537 100644 --- a/models/base/google/reservations/information_schema_reservations.sql +++ b/models/base/google/reservations/information_schema_reservations.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservations -#} SELECT ddl, project_id, project_number, reservation_name, ignore_idle_slots, slot_capacity, target_job_concurrency, autoscale, edition diff --git a/models/base/google/reservations/information_schema_reservations_timeline.sql b/models/base/google/reservations/information_schema_reservations_timeline.sql index ccf4f37..45d3d33 100644 --- a/models/base/google/reservations/information_schema_reservations_timeline.sql +++ b/models/base/google/reservations/information_schema_reservations_timeline.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservation-timeline -#} SELECT autoscale, edition, ignore_idle_slots, period_start, project_id, project_number, reservation_id, reservation_name, slots_assigned, slots_max_assigned diff --git a/models/base/google/routines/information_schema_parameters.sql b/models/base/google/routines/information_schema_parameters.sql index f076386..4a16423 100644 --- a/models/base/google/routines/information_schema_parameters.sql +++ b/models/base/google/routines/information_schema_parameters.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-parameters -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.PARAMETERS view, you need the following Identity and Access Management (IAM) permissions: diff --git a/models/base/google/routines/information_schema_routine_options.sql b/models/base/google/routines/information_schema_routine_options.sql index 7cee6ea..a836c5e 100644 --- a/models/base/google/routines/information_schema_routine_options.sql +++ b/models/base/google/routines/information_schema_routine_options.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-routine-options -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.ROUTINE_OPTIONS view, you need the following Identity and Access Management (IAM) permissions: diff --git a/models/base/google/routines/information_schema_routines.sql b/models/base/google/routines/information_schema_routines.sql index fdeb4a3..5cf348e 100644 --- a/models/base/google/routines/information_schema_routines.sql +++ b/models/base/google/routines/information_schema_routines.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-routines -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.ROUTINES view, you need the following Identity and Access Management (IAM) permissions: diff --git a/models/base/google/search_indexes/information_schema_search_index_columns.sql b/models/base/google/search_indexes/information_schema_search_index_columns.sql index 1c55b66..ca8f0da 100644 --- a/models/base/google/search_indexes/information_schema_search_index_columns.sql +++ b/models/base/google/search_indexes/information_schema_search_index_columns.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-index-columns -#} {# Required role/permissions: To see search index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) diff --git a/models/base/google/search_indexes/information_schema_search_indexes.sql b/models/base/google/search_indexes/information_schema_search_indexes.sql index 6b784b2..f3bfa3d 100644 --- a/models/base/google/search_indexes/information_schema_search_indexes.sql +++ b/models/base/google/search_indexes/information_schema_search_indexes.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-indexes -#} {# Required role/permissions: To see search index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) diff --git a/models/base/google/sessions/information_schema_sessions_by_project.sql b/models/base/google/sessions/information_schema_sessions_by_project.sql index c1b1fe7..78d2262 100644 --- a/models/base/google/sessions/information_schema_sessions_by_project.sql +++ b/models/base/google/sessions/information_schema_sessions_by_project.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-sessions-by-project -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.SESSIONS_BY_PROJECT view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the project. diff --git a/models/base/google/sessions/information_schema_sessions_by_user.sql b/models/base/google/sessions/information_schema_sessions_by_user.sql index 3fb67a0..f0dff51 100644 --- a/models/base/google/sessions/information_schema_sessions_by_user.sql +++ b/models/base/google/sessions/information_schema_sessions_by_user.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-sessions-by-user -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.SESSIONS_BY_USER view, you need the bigquery.jobs.list Identity and Access Management (IAM) permission for the project. diff --git a/models/base/google/streaming/information_schema_streaming_timeline.sql b/models/base/google/streaming/information_schema_streaming_timeline.sql index 5ef35b0..b00cc93 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline.sql +++ b/models/base/google/streaming/information_schema_streaming_timeline.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming -#} SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes diff --git a/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql b/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql index 74111b9..262ef27 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql +++ b/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming-by-folder -#} SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes diff --git a/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql b/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql index 5d99202..118cec6 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql +++ b/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming-by-organization -#} SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes diff --git a/models/base/google/tables/information_schema_column_field_paths.sql b/models/base/google/tables/information_schema_column_field_paths.sql index d1f2f04..77c37a0 100644 --- a/models/base/google/tables/information_schema_column_field_paths.sql +++ b/models/base/google/tables/information_schema_column_field_paths.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-column-field-paths -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.COLUMN_FIELD_PATHS view, you need the following Identity and Access Management (IAM) permissions: diff --git a/models/base/google/tables/information_schema_columns.sql b/models/base/google/tables/information_schema_columns.sql index 7c48ba8..d3a4a82 100644 --- a/models/base/google/tables/information_schema_columns.sql +++ b/models/base/google/tables/information_schema_columns.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-columns -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.COLUMNS view, you need the following Identity and Access Management (IAM) permissions: diff --git a/models/base/google/tables/information_schema_table_constraints.sql b/models/base/google/tables/information_schema_table_constraints.sql index cf824bb..c180b75 100644 --- a/models/base/google/tables/information_schema_table_constraints.sql +++ b/models/base/google/tables/information_schema_table_constraints.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-constraints -#} {# Required role/permissions: You need the following Identity and Access Management (IAM) permissions: diff --git a/models/base/google/tables/information_schema_table_options.sql b/models/base/google/tables/information_schema_table_options.sql index c7c2c4c..d6e6a69 100644 --- a/models/base/google/tables/information_schema_table_options.sql +++ b/models/base/google/tables/information_schema_table_options.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-options -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_OPTIONS view, you need the following Identity and Access Management (IAM) permissions: diff --git a/models/base/google/tables/information_schema_table_snapshots.sql b/models/base/google/tables/information_schema_table_snapshots.sql index 829b21d..978392c 100644 --- a/models/base/google/tables/information_schema_table_snapshots.sql +++ b/models/base/google/tables/information_schema_table_snapshots.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-snapshots -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_SNAPSHOTS view, you need the bigquery.tables.list Identity and Access Management (IAM) permission for the dataset. diff --git a/models/base/google/tables/information_schema_table_storage.sql b/models/base/google/tables/information_schema_table_storage.sql index bb290cb..e43a455 100644 --- a/models/base/google/tables/information_schema_table_storage.sql +++ b/models/base/google/tables/information_schema_table_storage.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage -#} SELECT project_id, project_number, table_catalog, table_schema, table_name, creation_time, total_rows, total_partitions, total_logical_bytes, active_logical_bytes, long_term_logical_bytes, current_physical_bytes, total_physical_bytes, active_physical_bytes, long_term_physical_bytes, time_travel_physical_bytes, storage_last_modified_time, deleted, table_type, fail_safe_physical_bytes, last_metadata_index_refresh_time diff --git a/models/base/google/tables/information_schema_table_storage_by_organization.sql b/models/base/google/tables/information_schema_table_storage_by_organization.sql index 5f16ddf..e0549d0 100644 --- a/models/base/google/tables/information_schema_table_storage_by_organization.sql +++ b/models/base/google/tables/information_schema_table_storage_by_organization.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-by-organization -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_BY_ORGANIZATION view, you need the following Identity and Access Management (IAM) permissions for your organization: diff --git a/models/base/google/tables/information_schema_table_storage_usage_timeline.sql b/models/base/google/tables/information_schema_table_storage_usage_timeline.sql index ef1d427..328397a 100644 --- a/models/base/google/tables/information_schema_table_storage_usage_timeline.sql +++ b/models/base/google/tables/information_schema_table_storage_usage_timeline.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-usage -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_USAGE_TIMELINE view, you need the following Identity and Access Management (IAM) permissions: diff --git a/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql b/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql index 43569a5..aa744b5 100644 --- a/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql +++ b/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-usage-by-organization -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_USAGE_TIMELINE_BY_ORGANIZATION view, you need the following Identity and Access Management (IAM) permissions for your organization: diff --git a/models/base/google/tables/information_schema_tables.sql b/models/base/google/tables/information_schema_tables.sql index c67a459..4246d67 100644 --- a/models/base/google/tables/information_schema_tables.sql +++ b/models/base/google/tables/information_schema_tables.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-tables -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLES view, you need the following Identity and Access Management (IAM) permissions: diff --git a/models/base/google/vector_indexes/information_schema_vector_index_columns.sql b/models/base/google/vector_indexes/information_schema_vector_index_columns.sql index a893fff..15bbc57 100644 --- a/models/base/google/vector_indexes/information_schema_vector_index_columns.sql +++ b/models/base/google/vector_indexes/information_schema_vector_index_columns.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-index-columns -#} {# Required role/permissions: To see vector index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) diff --git a/models/base/google/vector_indexes/information_schema_vector_index_options.sql b/models/base/google/vector_indexes/information_schema_vector_index_options.sql index 63bb469..dbf38ab 100644 --- a/models/base/google/vector_indexes/information_schema_vector_index_options.sql +++ b/models/base/google/vector_indexes/information_schema_vector_index_options.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-index-options -#} {# Required role/permissions: To see vector index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) diff --git a/models/base/google/vector_indexes/information_schema_vector_indexes.sql b/models/base/google/vector_indexes/information_schema_vector_indexes.sql index 5bc180f..1f11669 100644 --- a/models/base/google/vector_indexes/information_schema_vector_indexes.sql +++ b/models/base/google/vector_indexes/information_schema_vector_indexes.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-indexes -#} {# Required role/permissions: To see vector index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) diff --git a/models/base/google/views/information_schema_materialized_views.sql b/models/base/google/views/information_schema_materialized_views.sql index 8686c00..c4d1ca0 100644 --- a/models/base/google/views/information_schema_materialized_views.sql +++ b/models/base/google/views/information_schema_materialized_views.sql @@ -1,6 +1,6 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-materialized-views -#} -{# Required role/permissions: +{# Required role/permissions: To get the permissions that you need to query the INFORMATION_SCHEMA.MATERIALIZED_VIEWS view, @@ -28,7 +28,7 @@ Required permissions The following permissions are required to query the INFORMATION_SCHEMA.MATERIALIZED_VIEWS view: - bigquery.tables.get + bigquery.tables.get bigquery.tables.list diff --git a/models/base/google/views/information_schema_views.sql b/models/base/google/views/information_schema_views.sql index 544bac2..f8ca33b 100644 --- a/models/base/google/views/information_schema_views.sql +++ b/models/base/google/views/information_schema_views.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-views -#} {# Required role/permissions: To get view metadata, you need the following Identity and Access Management (IAM) permissions: diff --git a/models/base/google/write_api/information_schema_write_api_timeline.sql b/models/base/google/write_api/information_schema_write_api_timeline.sql index fd27a52..7624281 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline.sql +++ b/models/base/google/write_api/information_schema_write_api_timeline.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api -#} SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes diff --git a/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql b/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql index 4ac41b8..ac89678 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql +++ b/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api-by-folder -#} SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes diff --git a/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql b/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql index 7d81704..f7afdfb 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql +++ b/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql @@ -1,4 +1,4 @@ -{{ config(materialization='project_by_project_table') }} +{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api-by-organization -#} SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes From a179a6e79d993f5320d8dd4beb8cdb2cc398b8f4 Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Sun, 12 Jan 2025 13:14:35 +0100 Subject: [PATCH 05/12] clean up materialization --- macros/project_by_project_table.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/macros/project_by_project_table.sql b/macros/project_by_project_table.sql index 5b40ace..9a1a996 100644 --- a/macros/project_by_project_table.sql +++ b/macros/project_by_project_table.sql @@ -2,7 +2,6 @@ {% set target_relation = this %} {% set existing_relation = load_relation(this) %} -{% set tmp_relation = make_temp_relation(this) %} {% set projects = project_list() %} {%- set raw_partition_by = config.get('partition_by', none) -%} {%- set partition_config = adapter.parse_partition_by(raw_partition_by) -%} @@ -83,6 +82,10 @@ {{ log("Materialization complete", info=True) }} {{ run_hooks(post_hooks) }} +{% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %} +{% do apply_grants(target_relation, grant_config, should_revoke) %} +{%- set full_refresh_mode = (should_full_refresh()) -%} +{% do persist_docs(target_relation, model) %} {{ return({'relations': [target_relation]}) }} From b888d17389267833c701e8b01a8a0f0cfadc61a9 Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Sun, 12 Jan 2025 13:17:25 +0100 Subject: [PATCH 06/12] remove debug logs --- macros/project_by_project_table.sql | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/macros/project_by_project_table.sql b/macros/project_by_project_table.sql index 9a1a996..092b3ce 100644 --- a/macros/project_by_project_table.sql +++ b/macros/project_by_project_table.sql @@ -5,7 +5,6 @@ {% set projects = project_list() %} {%- set raw_partition_by = config.get('partition_by', none) -%} {%- set partition_config = adapter.parse_partition_by(raw_partition_by) -%} -{{ log("project_by_project_table with partition_by : " ~ partition_by, info=True) }} {{ run_hooks(pre_hooks) }} @@ -13,47 +12,36 @@ {% if existing_relation is none %} {% call statement('main') -%} {% if partition_config is not none %} - {{ log("Creating table with partition_by : " ~ partition_by, info=True) }} {% set build_sql = create_table_as(False, target_relation, sql) %} {% else %} - {{ log("Creating table without partition_by", info=True) }} {% set build_sql = create_table_as(False, target_relation, sql) %} {% endif %} - {{ log("Running build_sql : " ~ build_sql, info=True) }} {{ build_sql }} {%- endcall %} {% else %} {% call statement('main') -%} SELECT 1 {%- endcall %} - {{ log("Table already exists", info=True) }} {% if partition_config is not none %} -- Get the maximum partition value {% set max_partition_sql %} SELECT MAX({{ partition_config.field }}) as max_partition FROM {{ target_relation }} {% endset %} - {{ log("Running max_partition_sql : " ~ max_partition_sql, info=True) }} {% else %} -- Truncate the table if partition_by is not defined {% set truncate_sql %} TRUNCATE TABLE {{ target_relation }} {% endset %} - {{ log("Running truncate_sql : " ~ truncate_sql, info=True) }} {{ truncate_sql }} {% do run_query(truncate_sql) %} {% endif %} - {{ log("Check partition_config", info=True) }} {% if partition_config is not none %} - {{ log("Getting max_partition_value", info=True) }} {% set max_partition_result = run_query(max_partition_sql) %} - {{ log("max_partition_result : " ~ max_partition_result, info=True) }} {% if max_partition_result['data']|length > 0 %} {% set max_partition_value = max_partition_result['data'][0]['max_partition'] %} - {{ log("max_partition_value : " ~ max_partition_value, info=True) }} {% else %} {% set max_partition_value = none %} - {{ log("No max_partition_value found", info=True) }} {% endif %} {% endif %} {% endif %} @@ -62,7 +50,6 @@ {% if projects|length > 0 %} {% set all_insert_sql = [] %} {% for project in projects %} - {{ log("Processing project : " ~ project, info=True) }} {% set project_sql = sql | replace('`region-', '`' ~ project | trim ~ '`.`region-') %} {% if existing_relation is not none and partition_config is not none and max_partition_value is not none %} {% set project_sql = project_sql + ' WHERE ' ~ partition_config.field ~ ' >= "' ~ max_partition_value ~ '"' %} @@ -71,16 +58,13 @@ INSERT INTO {{ target_relation }} {{ project_sql }} {% endset %} - {{ log("Running insert_sql : " ~ insert_sql, info=True) }} {% do all_insert_sql.append(insert_sql) %} {% endfor %} - {{ log(" all_insert_sql : " ~ all_insert_sql, info=True) }} {% call statement('insert') -%} {{ all_insert_sql | join(';\n') }} {%- endcall %} {% endif %} -{{ log("Materialization complete", info=True) }} {{ run_hooks(post_hooks) }} {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %} {% do apply_grants(target_relation, grant_config, should_revoke) %} From 174bf7a3d8bcfacd7a7d150e5a54314e72e77725 Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Sun, 12 Jan 2025 13:45:19 +0100 Subject: [PATCH 07/12] Allow to override the materialization for information schema tables --- dbt_project.yml | 7 ++++--- macros/dbt_bigquery_monitoring_materalization.sql | 9 --------- macros/materalization_information_schema.sql | 12 ++++++++++++ 3 files changed, 16 insertions(+), 12 deletions(-) delete mode 100644 macros/dbt_bigquery_monitoring_materalization.sql create mode 100644 macros/materalization_information_schema.sql diff --git a/dbt_project.yml b/dbt_project.yml index 7569e5f..92afea8 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -25,7 +25,7 @@ models: - "dbt-bigquery-monitoring-storage" base: google: - +materialized: "ephemeral" + +materialized: "{{ var('google_information_schema_model_materialization', 'placeholder') if var('google_information_schema_model_materialization', 'placeholder') != 'placeholder' else 'ephemeral' }}" vars: # Environment configuration @@ -51,7 +51,7 @@ vars: # Project input configuration # The number of days to look back for regular tables, you might use up to 180 days usually. # Expiration on intermediate tables is aligned so that it can store data as old your maximum lookback window as it's partitioned by time. - lookback_window_days: "{{ env_var('DBT_BQ_MONITORING_LOOKBACK_WINDOW_DAYS', 7) }}" + lookback_window_days: "{{ env_var('DBT_BQ_MONITORING_LOOKBACK_WINDOW_DAYS', 7) }}" # Billing data can be late, a safe window is to refresh data for the past 3 days but you can increase it for exceptional cases lookback_incremental_billing_window_days: "{{ env_var('DBT_BQ_MONITORING_LOOKBACK_INCREMENTAL_BILLING_WINDOW_DAYS', 3) }}" # Project output configuration @@ -59,7 +59,8 @@ vars: output_limit_size: "{{ env_var('DBT_BQ_MONITORING_OUTPUT_LIMIT_SIZE', 1000) }}" output_partition_expiration_days: "{{ env_var('DBT_BQ_MONITORING_TABLE_EXPIRATION_DAYS', 365) }}" use_copy_partitions: "{{ env_var('DBT_BQ_MONITORING_USE_COPY_PARTITIONS', true) }}" - + google_information_schema_model_materialization: "{{ env_var('DBT_BQ_MONITORING_GOOGLE_INFORMATION_SCHEMA_MODELS_MATERIALIZATION', 'placeholder') }}" + # GCP Billing export (required for storage cost monitoring over time) # The values are configured during the export setup in the GCP Console enable_gcp_billing_export: "{{ env_var('DBT_BQ_MONITORING_ENABLE_GCP_BILLING_EXPORT', false) }}" diff --git a/macros/dbt_bigquery_monitoring_materalization.sql b/macros/dbt_bigquery_monitoring_materalization.sql deleted file mode 100644 index eca8d8e..0000000 --- a/macros/dbt_bigquery_monitoring_materalization.sql +++ /dev/null @@ -1,9 +0,0 @@ -{% macro dbt_bigquery_monitoring_materialization() %} - {% set projects = project_list() %} - {% if projects|length == 0 %} - {% set materialization = 'ephemeral' %} - {% else %} - {% set materialization = 'project_by_project_table' %} - {% endif %} - {{ return(materialization) }} -{% endmacro %} diff --git a/macros/materalization_information_schema.sql b/macros/materalization_information_schema.sql new file mode 100644 index 0000000..bc2b572 --- /dev/null +++ b/macros/materalization_information_schema.sql @@ -0,0 +1,12 @@ +{% macro dbt_bigquery_monitoring_materialization() %} +{% set projects = project_list() %} + {#- If the user has set the materialization in the config that's different from the default -#} + {% if var('google_information_schema_model_materialization') != 'placeholder' %} + {% set materialization = var('google_information_schema_model_materialization') %} + {% elif projects|length == 0 %} + {% set materialization = 'ephemeral' %} + {% else %} + {% set materialization = 'project_by_project_table' %} + {% endif %} + {{ return(materialization) }} +{% endmacro %} From ef604143f6ffd66fb74ef9ff73595d068a49b9e7 Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Sun, 12 Jan 2025 14:00:05 +0100 Subject: [PATCH 08/12] add table expiration --- macros/project_by_project_table.sql | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/macros/project_by_project_table.sql b/macros/project_by_project_table.sql index 092b3ce..4348ae9 100644 --- a/macros/project_by_project_table.sql +++ b/macros/project_by_project_table.sql @@ -5,14 +5,24 @@ {% set projects = project_list() %} {%- set raw_partition_by = config.get('partition_by', none) -%} {%- set partition_config = adapter.parse_partition_by(raw_partition_by) -%} +{%- set full_refresh_mode = (should_full_refresh()) -%} {{ run_hooks(pre_hooks) }} --- Create the table if it doesn't exist -{% if existing_relation is none %} +-- Create the table if it doesn't exist or if we're in full-refresh mode +{% if existing_relation is none or full_refresh_mode %} {% call statement('main') -%} {% if partition_config is not none %} {% set build_sql = create_table_as(False, target_relation, sql) %} + -- Add partition expiration to the table to 180 days like information_schema tables + {% set partition_expiration_sql %} + ALTER TABLE {{ target_relation }} + SET OPTIONS ( + expiration_timestamp = 180 + ) + {% endset %} + {{ partition_expiration_sql }} + {% do run_query(partition_expiration_sql) %} {% else %} {% set build_sql = create_table_as(False, target_relation, sql) %} {% endif %} @@ -68,7 +78,6 @@ {{ run_hooks(post_hooks) }} {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %} {% do apply_grants(target_relation, grant_config, should_revoke) %} -{%- set full_refresh_mode = (should_full_refresh()) -%} {% do persist_docs(target_relation, model) %} {{ return({'relations': [target_relation]}) }} From 58dc0c1ec08b75b64fcb5146620d8d7b3bf4847c Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Sun, 12 Jan 2025 20:14:32 +0100 Subject: [PATCH 09/12] fix timestamp --- macros/project_by_project_table.sql | 29 ++++++++----------- .../google/jobs/information_schema_jobs.sql | 2 +- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/macros/project_by_project_table.sql b/macros/project_by_project_table.sql index 4348ae9..6d08e51 100644 --- a/macros/project_by_project_table.sql +++ b/macros/project_by_project_table.sql @@ -9,22 +9,15 @@ {{ run_hooks(pre_hooks) }} +{%- set sql_no_data = sql + " LIMIT 0" %} + -- Create the table if it doesn't exist or if we're in full-refresh mode {% if existing_relation is none or full_refresh_mode %} {% call statement('main') -%} {% if partition_config is not none %} - {% set build_sql = create_table_as(False, target_relation, sql) %} - -- Add partition expiration to the table to 180 days like information_schema tables - {% set partition_expiration_sql %} - ALTER TABLE {{ target_relation }} - SET OPTIONS ( - expiration_timestamp = 180 - ) - {% endset %} - {{ partition_expiration_sql }} - {% do run_query(partition_expiration_sql) %} + {% set build_sql = create_table_as(False, target_relation, sql_no_data) %} {% else %} - {% set build_sql = create_table_as(False, target_relation, sql) %} + {% set build_sql = create_table_as(False, target_relation, sql_no_data) %} {% endif %} {{ build_sql }} {%- endcall %} @@ -35,8 +28,9 @@ {% if partition_config is not none %} -- Get the maximum partition value {% set max_partition_sql %} - SELECT MAX({{ partition_config.field }}) as max_partition + SELECT FORMAT_TIMESTAMP("%F %T", MAX({{ partition_config.field }})) as max_partition FROM {{ target_relation }} + WHERE {{ partition_config.field }} IS NOT NULL {% endset %} {% else %} -- Truncate the table if partition_by is not defined @@ -48,10 +42,8 @@ {% endif %} {% if partition_config is not none %} {% set max_partition_result = run_query(max_partition_sql) %} - {% if max_partition_result['data']|length > 0 %} - {% set max_partition_value = max_partition_result['data'][0]['max_partition'] %} - {% else %} - {% set max_partition_value = none %} + {% if max_partition_result|length > 0 %} + {% set max_partition_value = max_partition_result.columns[0].values()[0] %} {% endif %} {% endif %} {% endif %} @@ -61,8 +53,11 @@ {% set all_insert_sql = [] %} {% for project in projects %} {% set project_sql = sql | replace('`region-', '`' ~ project | trim ~ '`.`region-') %} - {% if existing_relation is not none and partition_config is not none and max_partition_value is not none %} + {% if existing_relation is not none and partition_config is not none and max_partition_value is not none and max_partition_value | length > 0 %} {% set project_sql = project_sql + ' WHERE ' ~ partition_config.field ~ ' >= "' ~ max_partition_value ~ '"' %} + {% else %} + {#- bigquery doesn't allow more than 4000 partitions per insert so if we have hourly tables it's ~ 166 days -#} + {% set project_sql = project_sql + ' WHERE ' ~ partition_config.field ~ ' >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 166 DAY)' %} {% endif %} {% set insert_sql %} INSERT INTO {{ target_relation }} diff --git a/models/base/google/jobs/information_schema_jobs.sql b/models/base/google/jobs/information_schema_jobs.sql index c79f852..6222048 100644 --- a/models/base/google/jobs/information_schema_jobs.sql +++ b/models/base/google/jobs/information_schema_jobs.sql @@ -1,4 +1,4 @@ -{{ config(materialized=dbt_bigquery_monitoring_materialization(), partition_by={'field': 'creation_time', 'data_type': 'timestamp', 'granularity': 'hour'}) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization(), partition_by={'field': 'creation_time', 'data_type': 'timestamp', 'granularity': 'hour'}, partition_expiration_days=180) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs -#} {# Required role/permissions: From 2ae4a28ae50fcf2381f088e881b22ea3fbe470c9 Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Mon, 13 Jan 2025 00:15:46 +0100 Subject: [PATCH 10/12] proper incremental on project_by_project_table, fix formatting and fix bi_statistics struct combination --- macros/project_by_project_table.sql | 18 +++-- models/base/combined_jobs_inputs.sql | 2 +- .../information_schema_object_privileges.sql | 10 ++- .../information_schema_bi_capacities.sql | 9 ++- ...information_schema_bi_capacity_changes.sql | 11 ++- ...ation_schema_effective_project_options.sql | 9 ++- ...nformation_schema_organization_options.sql | 8 +- ...on_schema_organization_options_changes.sql | 9 ++- .../information_schema_project_options.sql | 8 +- ...rmation_schema_project_options_changes.sql | 9 ++- .../datasets/information_schema_schemata.sql | 12 ++- .../information_schema_schemata_links.sql | 11 ++- .../information_schema_schemata_options.sql | 9 ++- .../information_schema_schemata_replicas.sql | 16 +++- ...emata_replicas_by_failover_reservation.sql | 16 +++- ...nformation_schema_shared_dataset_usage.sql | 20 ++++- .../google/jobs/information_schema_jobs.sql | 37 ++++++++- .../information_schema_jobs_by_folder.sql | 39 +++++++++- ...nformation_schema_jobs_by_organization.sql | 39 +++++++++- .../information_schema_jobs_by_project.sql | 39 +++++++++- .../jobs/information_schema_jobs_by_user.sql | 40 +++++++++- .../information_schema_jobs_timeline.sql | 27 ++++++- ...rmation_schema_jobs_timeline_by_folder.sql | 28 ++++++- ...n_schema_jobs_timeline_by_organization.sql | 27 ++++++- ...formation_schema_jobs_timeline_by_user.sql | 27 ++++++- .../information_schema_insights.sql | 17 +++- .../information_schema_recommendations.sql | 17 +++- ...schema_recommendations_by_organization.sql | 17 +++- .../information_schema_assignment_changes.sql | 16 +++- .../information_schema_assignments.sql | 13 +++- ...ion_schema_capacity_commitment_changes.sql | 19 ++++- ...nformation_schema_capacity_commitments.sql | 14 +++- ...information_schema_reservation_changes.sql | 15 +++- .../information_schema_reservations.sql | 13 +++- ...formation_schema_reservations_timeline.sql | 14 +++- .../information_schema_parameters.sql | 14 +++- .../information_schema_routine_options.sql | 10 ++- .../routines/information_schema_routines.sql | 21 ++++- ...nformation_schema_search_index_columns.sql | 10 ++- .../information_schema_search_indexes.sql | 20 ++++- ...information_schema_sessions_by_project.sql | 12 ++- .../information_schema_sessions_by_user.sql | 12 ++- .../information_schema_streaming_timeline.sql | 13 +++- ...on_schema_streaming_timeline_by_folder.sql | 13 +++- ...ema_streaming_timeline_by_organization.sql | 13 +++- .../information_schema_column_field_paths.sql | 13 +++- .../tables/information_schema_columns.sql | 22 +++++- ...rmation_schema_constraint_column_usage.sql | 74 ++++++++++-------- .../information_schema_key_column_usage.sql | 78 +++++++++++-------- .../tables/information_schema_partitions.sql | 78 +++++++++++-------- .../information_schema_table_constraints.sql | 14 +++- .../information_schema_table_options.sql | 10 ++- .../information_schema_table_snapshots.sql | 11 ++- .../information_schema_table_storage.sql | 25 +++++- ...n_schema_table_storage_by_organization.sql | 25 +++++- ...on_schema_table_storage_usage_timeline.sql | 15 +++- ...storage_usage_timeline_by_organization.sql | 15 +++- .../tables/information_schema_tables.sql | 24 +++++- ...nformation_schema_vector_index_columns.sql | 10 ++- ...nformation_schema_vector_index_options.sql | 11 ++- .../information_schema_vector_indexes.sql | 19 ++++- .../information_schema_materialized_views.sql | 14 +++- .../google/views/information_schema_views.sql | 10 ++- .../information_schema_write_api_timeline.sql | 14 +++- ...on_schema_write_api_timeline_by_folder.sql | 14 +++- ...ema_write_api_timeline_by_organization.sql | 14 +++- models/base/jobs_from_audit_logs.sql | 2 - 67 files changed, 1054 insertions(+), 231 deletions(-) diff --git a/macros/project_by_project_table.sql b/macros/project_by_project_table.sql index 6d08e51..ed67924 100644 --- a/macros/project_by_project_table.sql +++ b/macros/project_by_project_table.sql @@ -54,15 +54,23 @@ {% for project in projects %} {% set project_sql = sql | replace('`region-', '`' ~ project | trim ~ '`.`region-') %} {% if existing_relation is not none and partition_config is not none and max_partition_value is not none and max_partition_value | length > 0 %} - {% set project_sql = project_sql + ' WHERE ' ~ partition_config.field ~ ' >= "' ~ max_partition_value ~ '"' %} + {% set where_condition = 'WHERE ' ~ partition_config.field ~ ' >= TIMESTAMP_TRUNC("' ~ max_partition_value ~ '", HOUR)' %} + {% set insert_sql %} + DELETE FROM {{ target_relation }} + {{ where_condition }}; + + INSERT INTO {{ target_relation }} + {{ project_sql }} + {{ where_condition }} + {% endset %} {% else %} {#- bigquery doesn't allow more than 4000 partitions per insert so if we have hourly tables it's ~ 166 days -#} {% set project_sql = project_sql + ' WHERE ' ~ partition_config.field ~ ' >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 166 DAY)' %} + {% set insert_sql %} + INSERT INTO {{ target_relation }} + {{ project_sql }} + {% endset %} {% endif %} - {% set insert_sql %} - INSERT INTO {{ target_relation }} - {{ project_sql }} - {% endset %} {% do all_insert_sql.append(insert_sql) %} {% endfor %} {% call statement('insert') -%} diff --git a/models/base/combined_jobs_inputs.sql b/models/base/combined_jobs_inputs.sql index 8fbf07d..a43a263 100644 --- a/models/base/combined_jobs_inputs.sql +++ b/models/base/combined_jobs_inputs.sql @@ -7,7 +7,7 @@ SELECT COALESCE(TIMESTAMP_TRUNC(a.timestamp, HOUR), TIMESTAMP_TRUNC(j.creation_time, HOUR)) AS hour, - COALESCE(a.bi_engine_statistics, j.bi_engine_statistics) AS bi_engine_statistics, + j.bi_engine_statistics AS bi_engine_statistics, -- this field is only available in information schema COALESCE(a.cache_hit, j.cache_hit) AS cache_hit, a.caller_supplied_user_agent AS caller_supplied_user_agent, -- this field is only available in the audit logs COALESCE(a.creation_time, j.creation_time) AS creation_time, diff --git a/models/base/google/access_control/information_schema_object_privileges.sql b/models/base/google/access_control/information_schema_object_privileges.sql index 3859130..3455040 100644 --- a/models/base/google/access_control/information_schema_object_privileges.sql +++ b/models/base/google/access_control/information_schema_object_privileges.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-object-privileges -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.OBJECT_PRIVILEGES view, you need following Identity and Access Management (IAM) permissions: @@ -7,5 +7,11 @@ bigquery.tables.getIamPolicy for tables and views. For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT object_catalog, object_schema, object_name, object_type, privilege_type, grantee +SELECT +object_catalog, +object_schema, +object_name, +object_type, +privilege_type, +grantee FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`OBJECT_PRIVILEGES` diff --git a/models/base/google/bi_engine/information_schema_bi_capacities.sql b/models/base/google/bi_engine/information_schema_bi_capacities.sql index 8ea7983..f811f2d 100644 --- a/models/base/google/bi_engine/information_schema_bi_capacities.sql +++ b/models/base/google/bi_engine/information_schema_bi_capacities.sql @@ -1,5 +1,10 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-bi-capacities -#} -SELECT project_id, project_number, bi_capacity_name, size, preferred_tables +SELECT +project_id, +project_number, +bi_capacity_name, +size, +preferred_tables FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITIES` diff --git a/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql b/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql index 7c8711b..92a9670 100644 --- a/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql +++ b/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql @@ -1,5 +1,12 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-bi-capacity-changes -#} -SELECT change_timestamp, project_id, project_number, bi_capacity_name, size, user_email, preferred_tables +SELECT +change_timestamp, +project_id, +project_number, +bi_capacity_name, +size, +user_email, +preferred_tables FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITY_CHANGES` diff --git a/models/base/google/configuration/information_schema_effective_project_options.sql b/models/base/google/configuration/information_schema_effective_project_options.sql index dbc24cf..53b8d13 100644 --- a/models/base/google/configuration/information_schema_effective_project_options.sql +++ b/models/base/google/configuration/information_schema_effective_project_options.sql @@ -6,5 +6,10 @@ roles/bigquery.jobUser For more information about granular BigQuery permissions, see roles and permissions. -#} -SELECT option_name, option_description, option_type, option_set_level, option_set_on_id -FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`EFFECTIVE_PROJECT_OPTIONS` \ No newline at end of file +SELECT +option_name, +option_description, +option_type, +option_set_level, +option_set_on_id +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`EFFECTIVE_PROJECT_OPTIONS` diff --git a/models/base/google/configuration/information_schema_organization_options.sql b/models/base/google/configuration/information_schema_organization_options.sql index 8f1d983..29ed9cd 100644 --- a/models/base/google/configuration/information_schema_organization_options.sql +++ b/models/base/google/configuration/information_schema_organization_options.sql @@ -7,5 +7,9 @@ roles/bigquery.jobUser For more information about granular BigQuery permissions, see roles and permissions. -#} -SELECT option_name, option_description, option_type, option_value -FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS` \ No newline at end of file +SELECT +option_name, +option_description, +option_type, +option_value +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS` diff --git a/models/base/google/configuration/information_schema_organization_options_changes.sql b/models/base/google/configuration/information_schema_organization_options_changes.sql index 6548d88..1cd2a01 100644 --- a/models/base/google/configuration/information_schema_organization_options_changes.sql +++ b/models/base/google/configuration/information_schema_organization_options_changes.sql @@ -15,5 +15,10 @@ -#} -SELECT update_time, username, updated_options, project_id, project_number -FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS_CHANGES` \ No newline at end of file +SELECT +update_time, +username, +updated_options, +project_id, +project_number +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS_CHANGES` diff --git a/models/base/google/configuration/information_schema_project_options.sql b/models/base/google/configuration/information_schema_project_options.sql index 3e57c9c..b8a160a 100644 --- a/models/base/google/configuration/information_schema_project_options.sql +++ b/models/base/google/configuration/information_schema_project_options.sql @@ -7,5 +7,9 @@ roles/bigquery.jobUser For more information about granular BigQuery permissions, see roles and permissions. -#} -SELECT option_name, option_description, option_type, option_value -FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS` \ No newline at end of file +SELECT +option_name, +option_description, +option_type, +option_value +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS` diff --git a/models/base/google/configuration/information_schema_project_options_changes.sql b/models/base/google/configuration/information_schema_project_options_changes.sql index 2c22dd8..ac8db23 100644 --- a/models/base/google/configuration/information_schema_project_options_changes.sql +++ b/models/base/google/configuration/information_schema_project_options_changes.sql @@ -5,5 +5,10 @@ IAM role roles/bigquery.admin includes the permissions that you need to create a configuration.For more information about granular BigQuery permissions, see roles and permissions. -#} -SELECT update_time, username, updated_options, project_id, project_number -FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS_CHANGES` \ No newline at end of file +SELECT +update_time, +username, +updated_options, +project_id, +project_number +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS_CHANGES` diff --git a/models/base/google/datasets/information_schema_schemata.sql b/models/base/google/datasets/information_schema_schemata.sql index b09389c..1ecb877 100644 --- a/models/base/google/datasets/information_schema_schemata.sql +++ b/models/base/google/datasets/information_schema_schemata.sql @@ -1,5 +1,13 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata -#} -SELECT catalog_name, schema_name, schema_owner, creation_time, last_modified_time, location, ddl, default_collation_name +SELECT +catalog_name, +schema_name, +schema_owner, +creation_time, +last_modified_time, +location, +ddl, +default_collation_name FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` diff --git a/models/base/google/datasets/information_schema_schemata_links.sql b/models/base/google/datasets/information_schema_schemata_links.sql index 83cc115..5cec401 100644 --- a/models/base/google/datasets/information_schema_schemata_links.sql +++ b/models/base/google/datasets/information_schema_schemata_links.sql @@ -1,5 +1,12 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata-links -#} -SELECT catalog_name, schema_name, linked_schema_catalog_number, linked_schema_catalog_name, linked_schema_name, linked_schema_creation_time, linked_schema_org_display_name +SELECT +catalog_name, +schema_name, +linked_schema_catalog_number, +linked_schema_catalog_name, +linked_schema_name, +linked_schema_creation_time, +linked_schema_org_display_name FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_LINKS` diff --git a/models/base/google/datasets/information_schema_schemata_options.sql b/models/base/google/datasets/information_schema_schemata_options.sql index 7dc3958..fc7f758 100644 --- a/models/base/google/datasets/information_schema_schemata_options.sql +++ b/models/base/google/datasets/information_schema_schemata_options.sql @@ -1,5 +1,10 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata-options -#} -SELECT catalog_name, schema_name, option_name, option_type, option_value +SELECT +catalog_name, +schema_name, +option_name, +option_type, +option_value FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_OPTIONS` diff --git a/models/base/google/datasets/information_schema_schemata_replicas.sql b/models/base/google/datasets/information_schema_schemata_replicas.sql index 6d36202..5f7a8e2 100644 --- a/models/base/google/datasets/information_schema_schemata_replicas.sql +++ b/models/base/google/datasets/information_schema_schemata_replicas.sql @@ -1,6 +1,6 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-schemata-replicas -#} -{# Required role/permissions: +{# Required role/permissions: To get the permissions that you need to query the INFORMATION_SCHEMA.SCHEMATA_REPLICAS view, @@ -21,5 +21,15 @@ roles. -#} -SELECT catalog_name, schema_name, replica_name, location, replica_primary_assigned, replica_primary_assignment_complete, creation_time, creation_complete, replication_time, sync_status +SELECT +catalog_name, +schema_name, +replica_name, +location, +replica_primary_assigned, +replica_primary_assignment_complete, +creation_time, +creation_complete, +replication_time, +sync_status FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_REPLICAS` diff --git a/models/base/google/datasets/information_schema_schemata_replicas_by_failover_reservation.sql b/models/base/google/datasets/information_schema_schemata_replicas_by_failover_reservation.sql index 476bb47..17de99d 100644 --- a/models/base/google/datasets/information_schema_schemata_replicas_by_failover_reservation.sql +++ b/models/base/google/datasets/information_schema_schemata_replicas_by_failover_reservation.sql @@ -20,5 +20,17 @@ roles. -#} -SELECT failover_reservation_project_id, failover_reservation_name, catalog_name, schema_name, replica_name, location, replica_primary_assigned, replica_primary_assignment_complete, creation_time, creation_complete, replication_time, sync_status -FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_REPLICAS_BY_FAILOVER_RESERVATION` \ No newline at end of file +SELECT +failover_reservation_project_id, +failover_reservation_name, +catalog_name, +schema_name, +replica_name, +location, +replica_primary_assigned, +replica_primary_assignment_complete, +creation_time, +creation_complete, +replication_time, +sync_status +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_REPLICAS_BY_FAILOVER_RESERVATION` diff --git a/models/base/google/datasets/information_schema_shared_dataset_usage.sql b/models/base/google/datasets/information_schema_shared_dataset_usage.sql index 4af9e52..8d41b9e 100644 --- a/models/base/google/datasets/information_schema_shared_dataset_usage.sql +++ b/models/base/google/datasets/information_schema_shared_dataset_usage.sql @@ -1,5 +1,21 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization(), partition_by={'field': 'job_start_time', 'data_type': 'timestamp', 'granularity': 'hour'}, partition_expiration_days=180) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-shared-dataset-usage -#} -SELECT project_id, dataset_id, table_id, data_exchange_id, listing_id, job_start_time, job_end_time, job_id, job_project_number, job_location, linked_project_number, linked_dataset_id, subscriber_org_number, subscriber_org_display_name, num_rows_processed, total_bytes_processed +SELECT +project_id, +dataset_id, +table_id, +data_exchange_id, +listing_id, +job_start_time, +job_end_time, +job_id, +job_project_number, +job_location, +linked_project_number, +linked_dataset_id, +subscriber_org_number, +subscriber_org_display_name, +num_rows_processed, +total_bytes_processed FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SHARED_DATASET_USAGE` diff --git a/models/base/google/jobs/information_schema_jobs.sql b/models/base/google/jobs/information_schema_jobs.sql index 6222048..c85afb2 100644 --- a/models/base/google/jobs/information_schema_jobs.sql +++ b/models/base/google/jobs/information_schema_jobs.sql @@ -1,6 +1,6 @@ {{ config(materialized=dbt_bigquery_monitoring_materialization(), partition_by={'field': 'creation_time', 'data_type': 'timestamp', 'granularity': 'hour'}, partition_expiration_days=180) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs -#} -{# Required role/permissions: +{# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS view, @@ -16,5 +16,38 @@ -#} -SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info +SELECT +bi_engine_statistics, +cache_hit, +creation_time, +destination_table, +end_time, +error_result, +job_id, +job_stages, +job_type, +labels, +parent_job_id, +priority, +project_id, +project_number, +query, +referenced_tables, +reservation_id, +edition, +session_info, +start_time, +state, +statement_type, +timeline, +total_bytes_billed, +total_bytes_processed, +total_modified_partitions, +total_slot_ms, +transaction_id, +user_email, +transferred_bytes, +materialized_view_statistics, +job_creation_reason, +query_info FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS` diff --git a/models/base/google/jobs/information_schema_jobs_by_folder.sql b/models/base/google/jobs/information_schema_jobs_by_folder.sql index a690dbd..930c472 100644 --- a/models/base/google/jobs/information_schema_jobs_by_folder.sql +++ b/models/base/google/jobs/information_schema_jobs_by_folder.sql @@ -1,6 +1,6 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization(), partition_by={'field': 'creation_time', 'data_type': 'timestamp', 'granularity': 'hour'}, partition_expiration_days=180) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-folder -#} -{# Required role/permissions: +{# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS_BY_FOLDER view, @@ -16,5 +16,38 @@ -#} -SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info +SELECT +bi_engine_statistics, +cache_hit, +creation_time, +destination_table, +end_time, +error_result, +job_id, +job_stages, +job_type, +labels, +parent_job_id, +priority, +project_id, +project_number, +query, +referenced_tables, +reservation_id, +edition, +session_info, +start_time, +state, +statement_type, +timeline, +total_bytes_billed, +total_bytes_processed, +total_modified_partitions, +total_slot_ms, +transaction_id, +user_email, +transferred_bytes, +materialized_view_statistics, +job_creation_reason, +query_info FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_FOLDER` diff --git a/models/base/google/jobs/information_schema_jobs_by_organization.sql b/models/base/google/jobs/information_schema_jobs_by_organization.sql index 98229b3..6222504 100644 --- a/models/base/google/jobs/information_schema_jobs_by_organization.sql +++ b/models/base/google/jobs/information_schema_jobs_by_organization.sql @@ -1,6 +1,6 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization(), partition_by={'field': 'creation_time', 'data_type': 'timestamp', 'granularity': 'hour'}, partition_expiration_days=180) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-organization -#} -{# Required role/permissions: +{# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS_BY_ORGANIZATION view, @@ -16,5 +16,38 @@ -#} -SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info +SELECT +bi_engine_statistics, +cache_hit, +creation_time, +destination_table, +end_time, +error_result, +job_id, +job_stages, +job_type, +labels, +parent_job_id, +priority, +project_id, +project_number, +query, +referenced_tables, +reservation_id, +edition, +session_info, +start_time, +state, +statement_type, +timeline, +total_bytes_billed, +total_bytes_processed, +total_modified_partitions, +total_slot_ms, +transaction_id, +user_email, +transferred_bytes, +materialized_view_statistics, +job_creation_reason, +query_info FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_ORGANIZATION` diff --git a/models/base/google/jobs/information_schema_jobs_by_project.sql b/models/base/google/jobs/information_schema_jobs_by_project.sql index 333657f..7485355 100644 --- a/models/base/google/jobs/information_schema_jobs_by_project.sql +++ b/models/base/google/jobs/information_schema_jobs_by_project.sql @@ -1,6 +1,6 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization(), partition_by={'field': 'creation_time', 'data_type': 'timestamp', 'granularity': 'hour'}, partition_expiration_days=180) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs -#} -{# Required role/permissions: +{# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS view, @@ -16,5 +16,38 @@ -#} -SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info +SELECT +bi_engine_statistics, +cache_hit, +creation_time, +destination_table, +end_time, +error_result, +job_id, +job_stages, +job_type, +labels, +parent_job_id, +priority, +project_id, +project_number, +query, +referenced_tables, +reservation_id, +edition, +session_info, +start_time, +state, +statement_type, +timeline, +total_bytes_billed, +total_bytes_processed, +total_modified_partitions, +total_slot_ms, +transaction_id, +user_email, +transferred_bytes, +materialized_view_statistics, +job_creation_reason, +query_info FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_PROJECT` diff --git a/models/base/google/jobs/information_schema_jobs_by_user.sql b/models/base/google/jobs/information_schema_jobs_by_user.sql index ed40d5e..e06c08c 100644 --- a/models/base/google/jobs/information_schema_jobs_by_user.sql +++ b/models/base/google/jobs/information_schema_jobs_by_user.sql @@ -1,6 +1,6 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization(), partition_by={'field': 'creation_time', 'data_type': 'timestamp', 'granularity': 'hour'}, partition_expiration_days=180) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-user -#} -{# Required role/permissions: +{# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS_BY_USER view, @@ -16,5 +16,39 @@ -#} -SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, edition, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info +SELECT +bi_engine_statistics, +cache_hit, +creation_time, +destination_table, +dml_statistics, +end_time, +error_result, +job_id, +job_stages, +job_type, +labels, +parent_job_id, +priority, +project_id, +project_number, +query, +referenced_tables, +reservation_id, +edition, +session_info, +start_time, +state, +statement_type, +timeline, +total_bytes_billed, +total_bytes_processed, +total_modified_partitions, +total_slot_ms, +transaction_id, +user_email, +transferred_bytes, +materialized_view_statistics, +job_creation_reason, +query_info FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_USER` diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql index 88279dd..df74814 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the project. @@ -9,5 +9,28 @@ BigQuery Admin For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT period_start, period_slot_ms, project_id, project_number, user_email, job_id, job_type, statement_type, priority, parent_job_id, job_creation_time, job_start_time, job_end_time, state, reservation_id, edition, total_bytes_billed, total_bytes_processed, error_result, cache_hit, period_shuffle_ram_usage_ratio, period_estimated_runnable_units, transaction_id +SELECT +period_start, +period_slot_ms, +project_id, +project_number, +user_email, +job_id, +job_type, +statement_type, +priority, +parent_job_id, +job_creation_time, +job_start_time, +job_end_time, +state, +reservation_id, +edition, +total_bytes_billed, +total_bytes_processed, +error_result, +cache_hit, +period_shuffle_ram_usage_ratio, +period_estimated_runnable_units, +transaction_id FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE` diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql index 003a5c6..2f7ef81 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-folder -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_FOLDER view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the parent @@ -9,5 +9,29 @@ BigQuery Admin For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT period_start, period_slot_ms, project_id, project_number, folder_numbers, user_email, job_id, job_type, statement_type, priority, parent_job_id, job_creation_time, job_start_time, job_end_time, state, reservation_id, edition, total_bytes_billed, total_bytes_processed, error_result, cache_hit, period_shuffle_ram_usage_ratio, period_estimated_runnable_units, transaction_id +SELECT +period_start, +period_slot_ms, +project_id, +project_number, +folder_numbers, +user_email, +job_id, +job_type, +statement_type, +priority, +parent_job_id, +job_creation_time, +job_start_time, +job_end_time, +state, +reservation_id, +edition, +total_bytes_billed, +total_bytes_processed, +error_result, +cache_hit, +period_shuffle_ram_usage_ratio, +period_estimated_runnable_units, +transaction_id FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_FOLDER` diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql index 6054980..f743531 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-organization -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_ORGANIZATION view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the organization. @@ -11,5 +11,28 @@ The JOBS_BY_ORGANIZATION schema table is only available to users with defined Google Cloud organizations.For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT period_start, period_slot_ms, project_id, project_number, folder_numbers, user_email, job_id, job_type, statement_type, priority, parent_job_id, job_creation_time, job_start_time, job_end_time, state, reservation_id, edition, total_bytes_billed, total_bytes_processed, error_result, cache_hit, period_shuffle_ram_usage_ratio, period_estimated_runnable_units +SELECT +period_start, +period_slot_ms, +project_id, +project_number, +folder_numbers, +user_email, +job_id, +job_type, +statement_type, +priority, +parent_job_id, +job_creation_time, +job_start_time, +job_end_time, +state, +reservation_id, +edition, +total_bytes_billed, +total_bytes_processed, +error_result, +cache_hit, +period_shuffle_ram_usage_ratio, +period_estimated_runnable_units FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_ORGANIZATION` diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql index 1cce2fa..b0880aa 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-user -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_USER view, you need the bigquery.jobs.list Identity and Access Management (IAM) permission for the project. @@ -9,5 +9,28 @@ BigQuery User For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT period_start, period_slot_ms, project_id, project_number, user_email, job_id, job_type, statement_type, priority, parent_job_id, job_creation_time, job_start_time, job_end_time, state, reservation_id, edition, total_bytes_billed, total_bytes_processed, error_result, cache_hit, period_shuffle_ram_usage_ratio, period_estimated_runnable_units, transaction_id +SELECT +period_start, +period_slot_ms, +project_id, +project_number, +user_email, +job_id, +job_type, +statement_type, +priority, +parent_job_id, +job_creation_time, +job_start_time, +job_end_time, +state, +reservation_id, +edition, +total_bytes_billed, +total_bytes_processed, +error_result, +cache_hit, +period_shuffle_ram_usage_ratio, +period_estimated_runnable_units, +transaction_id FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_USER` diff --git a/models/base/google/recommendations_and_insights/information_schema_insights.sql b/models/base/google/recommendations_and_insights/information_schema_insights.sql index ab0e438..194b9ef 100644 --- a/models/base/google/recommendations_and_insights/information_schema_insights.sql +++ b/models/base/google/recommendations_and_insights/information_schema_insights.sql @@ -1,5 +1,18 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-insights -#} -SELECT insight_id, insight_type, subtype, project_id, project_number, description, last_updated_time, category, target_resources, state, severity, associated_recommendation_ids, additional_details +SELECT +insight_id, +insight_type, +subtype, +project_id, +project_number, +description, +last_updated_time, +category, +target_resources, +state, +severity, +associated_recommendation_ids, +additional_details FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`INSIGHTS` diff --git a/models/base/google/recommendations_and_insights/information_schema_recommendations.sql b/models/base/google/recommendations_and_insights/information_schema_recommendations.sql index 2653347..a33588f 100644 --- a/models/base/google/recommendations_and_insights/information_schema_recommendations.sql +++ b/models/base/google/recommendations_and_insights/information_schema_recommendations.sql @@ -1,5 +1,18 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-recommendations -#} -SELECT recommendation_id, recommender, subtype, project_id, project_number, description, last_updated_time, target_resources, state, primary_impact, priority, associated_insight_ids, additional_details +SELECT +recommendation_id, +recommender, +subtype, +project_id, +project_number, +description, +last_updated_time, +target_resources, +state, +primary_impact, +priority, +associated_insight_ids, +additional_details FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RECOMMENDATIONS` diff --git a/models/base/google/recommendations_and_insights/information_schema_recommendations_by_organization.sql b/models/base/google/recommendations_and_insights/information_schema_recommendations_by_organization.sql index 9d80028..d27d92e 100644 --- a/models/base/google/recommendations_and_insights/information_schema_recommendations_by_organization.sql +++ b/models/base/google/recommendations_and_insights/information_schema_recommendations_by_organization.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-recommendations-by-org -#} {# Required role/permissions: To view recommendations with the INFORMATION_SCHEMA.RECOMMENDATIONS_BY_ORGANIZATION view, you must have the @@ -13,5 +13,18 @@ Materialized view recommendations permissions Role recommendations for datasets permissions -#} -SELECT recommendation_id, recommender, subtype, project_id, project_number, description, last_updated_time, target_resources, state, primary_impact, priority, associated_insight_ids, additional_details +SELECT +recommendation_id, +recommender, +subtype, +project_id, +project_number, +description, +last_updated_time, +target_resources, +state, +primary_impact, +priority, +associated_insight_ids, +additional_details FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RECOMMENDATIONS` diff --git a/models/base/google/reservations/information_schema_assignment_changes.sql b/models/base/google/reservations/information_schema_assignment_changes.sql index ac2e371..435d379 100644 --- a/models/base/google/reservations/information_schema_assignment_changes.sql +++ b/models/base/google/reservations/information_schema_assignment_changes.sql @@ -1,5 +1,17 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-assignments-changes -#} -SELECT change_timestamp, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type, action, user_email, state +SELECT +change_timestamp, +project_id, +project_number, +assignment_id, +reservation_name, +job_type, +assignee_id, +assignee_number, +assignee_type, +action, +user_email, +state FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENT_CHANGES` diff --git a/models/base/google/reservations/information_schema_assignments.sql b/models/base/google/reservations/information_schema_assignments.sql index 458fb4a..f2a358b 100644 --- a/models/base/google/reservations/information_schema_assignments.sql +++ b/models/base/google/reservations/information_schema_assignments.sql @@ -1,5 +1,14 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-assignments -#} -SELECT ddl, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type +SELECT +ddl, +project_id, +project_number, +assignment_id, +reservation_name, +job_type, +assignee_id, +assignee_number, +assignee_type FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENTS` diff --git a/models/base/google/reservations/information_schema_capacity_commitment_changes.sql b/models/base/google/reservations/information_schema_capacity_commitment_changes.sql index 885e529..916fb8b 100644 --- a/models/base/google/reservations/information_schema_capacity_commitment_changes.sql +++ b/models/base/google/reservations/information_schema_capacity_commitment_changes.sql @@ -1,5 +1,20 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-capacity-commitment-changes -#} -SELECT change_timestamp, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, action, user_email, commitment_start_time, commitment_end_time, failure_status, renewal_plan, edition, is_flat_rate +SELECT +change_timestamp, +project_id, +project_number, +capacity_commitment_id, +commitment_plan, +state, +slot_count, +action, +user_email, +commitment_start_time, +commitment_end_time, +failure_status, +renewal_plan, +edition, +is_flat_rate FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENT_CHANGES` diff --git a/models/base/google/reservations/information_schema_capacity_commitments.sql b/models/base/google/reservations/information_schema_capacity_commitments.sql index 7e9f864..87b9cd0 100644 --- a/models/base/google/reservations/information_schema_capacity_commitments.sql +++ b/models/base/google/reservations/information_schema_capacity_commitments.sql @@ -1,5 +1,15 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-capacity-commitments -#} -SELECT ddl, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, edition, is_flat_rate, renewal_plan +SELECT +ddl, +project_id, +project_number, +capacity_commitment_id, +commitment_plan, +state, +slot_count, +edition, +is_flat_rate, +renewal_plan FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENTS` diff --git a/models/base/google/reservations/information_schema_reservation_changes.sql b/models/base/google/reservations/information_schema_reservation_changes.sql index 5734d0f..7a33fa8 100644 --- a/models/base/google/reservations/information_schema_reservation_changes.sql +++ b/models/base/google/reservations/information_schema_reservation_changes.sql @@ -1,5 +1,16 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservation-changes -#} -SELECT change_timestamp, project_id, project_number, reservation_name, ignore_idle_slots, action, slot_capacity, user_email, target_job_concurrency, autoscale, edition +SELECT +change_timestamp, +project_id, +project_number, +reservation_name, +ignore_idle_slots, +action, +slot_capacity, +user_email, +target_job_concurrency, +autoscale, +edition FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATION_CHANGES` diff --git a/models/base/google/reservations/information_schema_reservations.sql b/models/base/google/reservations/information_schema_reservations.sql index c7b1537..5f122ea 100644 --- a/models/base/google/reservations/information_schema_reservations.sql +++ b/models/base/google/reservations/information_schema_reservations.sql @@ -1,5 +1,14 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservations -#} -SELECT ddl, project_id, project_number, reservation_name, ignore_idle_slots, slot_capacity, target_job_concurrency, autoscale, edition +SELECT +ddl, +project_id, +project_number, +reservation_name, +ignore_idle_slots, +slot_capacity, +target_job_concurrency, +autoscale, +edition FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS` diff --git a/models/base/google/reservations/information_schema_reservations_timeline.sql b/models/base/google/reservations/information_schema_reservations_timeline.sql index 45d3d33..a3d06c6 100644 --- a/models/base/google/reservations/information_schema_reservations_timeline.sql +++ b/models/base/google/reservations/information_schema_reservations_timeline.sql @@ -1,5 +1,15 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservation-timeline -#} -SELECT autoscale, edition, ignore_idle_slots, period_start, project_id, project_number, reservation_id, reservation_name, slots_assigned, slots_max_assigned +SELECT +autoscale, +edition, +ignore_idle_slots, +period_start, +project_id, +project_number, +reservation_id, +reservation_name, +slots_assigned, +slots_max_assigned FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS_TIMELINE` diff --git a/models/base/google/routines/information_schema_parameters.sql b/models/base/google/routines/information_schema_parameters.sql index 4a16423..c42e7a0 100644 --- a/models/base/google/routines/information_schema_parameters.sql +++ b/models/base/google/routines/information_schema_parameters.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-parameters -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.PARAMETERS view, you need the following Identity and Access Management (IAM) permissions: @@ -12,5 +12,15 @@ roles/bigquery.dataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT specific_catalog, specific_schema, specific_name, ordinal_position, parameter_mode, is_result, parameter_name, data_type, parameter_default, is_aggregate +SELECT +specific_catalog, +specific_schema, +specific_name, +ordinal_position, +parameter_mode, +is_result, +parameter_name, +data_type, +parameter_default, +is_aggregate FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PARAMETERS` diff --git a/models/base/google/routines/information_schema_routine_options.sql b/models/base/google/routines/information_schema_routine_options.sql index a836c5e..5d9af13 100644 --- a/models/base/google/routines/information_schema_routine_options.sql +++ b/models/base/google/routines/information_schema_routine_options.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-routine-options -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.ROUTINE_OPTIONS view, you need the following Identity and Access Management (IAM) permissions: @@ -12,5 +12,11 @@ roles/bigquery.dataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT specific_catalog, specific_schema, specific_name, option_name, option_type, option_value +SELECT +specific_catalog, +specific_schema, +specific_name, +option_name, +option_type, +option_value FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINE_OPTIONS` diff --git a/models/base/google/routines/information_schema_routines.sql b/models/base/google/routines/information_schema_routines.sql index 5cf348e..d761942 100644 --- a/models/base/google/routines/information_schema_routines.sql +++ b/models/base/google/routines/information_schema_routines.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-routines -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.ROUTINES view, you need the following Identity and Access Management (IAM) permissions: @@ -12,5 +12,22 @@ roles/bigquery.dataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT specific_catalog, specific_schema, specific_name, routine_catalog, routine_schema, routine_name, routine_type, data_type, routine_body, routine_definition, external_language, is_deterministic, security_type, created, last_altered, ddl, connection +SELECT +specific_catalog, +specific_schema, +specific_name, +routine_catalog, +routine_schema, +routine_name, +routine_type, +data_type, +routine_body, +routine_definition, +external_language, +is_deterministic, +security_type, +created, +last_altered, +ddl, +connection FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINES` diff --git a/models/base/google/search_indexes/information_schema_search_index_columns.sql b/models/base/google/search_indexes/information_schema_search_index_columns.sql index ca8f0da..013e8a7 100644 --- a/models/base/google/search_indexes/information_schema_search_index_columns.sql +++ b/models/base/google/search_indexes/information_schema_search_index_columns.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-index-columns -#} {# Required role/permissions: To see search index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) @@ -13,5 +13,11 @@ roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT index_catalog, index_schema, table_name, index_name, index_column_name, index_field_path +SELECT +index_catalog, +index_schema, +table_name, +index_name, +index_column_name, +index_field_path FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEX_COLUMNS` diff --git a/models/base/google/search_indexes/information_schema_search_indexes.sql b/models/base/google/search_indexes/information_schema_search_indexes.sql index f3bfa3d..a585737 100644 --- a/models/base/google/search_indexes/information_schema_search_indexes.sql +++ b/models/base/google/search_indexes/information_schema_search_indexes.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-indexes -#} {# Required role/permissions: To see search index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) @@ -13,5 +13,21 @@ roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT index_catalog, index_schema, table_name, index_name, index_status, creation_time, last_modification_time, last_refresh_time, disable_time, disable_reason, ddl, coverage_percentage, unindexed_row_count, total_logical_bytes, total_storage_bytes, analyzer +SELECT +index_catalog, +index_schema, +table_name, +index_name, +index_status, +creation_time, +last_modification_time, +last_refresh_time, +disable_time, +disable_reason, +ddl, +coverage_percentage, +unindexed_row_count, +total_logical_bytes, +total_storage_bytes, +analyzer FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEXES` diff --git a/models/base/google/sessions/information_schema_sessions_by_project.sql b/models/base/google/sessions/information_schema_sessions_by_project.sql index 78d2262..eec858a 100644 --- a/models/base/google/sessions/information_schema_sessions_by_project.sql +++ b/models/base/google/sessions/information_schema_sessions_by_project.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-sessions-by-project -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.SESSIONS_BY_PROJECT view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the project. @@ -9,5 +9,13 @@ BigQuery Admin For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT creation_time, expiration_time, is_active, last_modified_time, project_id, project_number, session_id, user_email +SELECT +creation_time, +expiration_time, +is_active, +last_modified_time, +project_id, +project_number, +session_id, +user_email FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_PROJECT` diff --git a/models/base/google/sessions/information_schema_sessions_by_user.sql b/models/base/google/sessions/information_schema_sessions_by_user.sql index f0dff51..166d355 100644 --- a/models/base/google/sessions/information_schema_sessions_by_user.sql +++ b/models/base/google/sessions/information_schema_sessions_by_user.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-sessions-by-user -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.SESSIONS_BY_USER view, you need the bigquery.jobs.list Identity and Access Management (IAM) permission for the project. @@ -9,5 +9,13 @@ BigQuery User For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT creation_time, expiration_time, is_active, last_modified_time, project_id, project_number, session_id, user_email +SELECT +creation_time, +expiration_time, +is_active, +last_modified_time, +project_id, +project_number, +session_id, +user_email FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_USER` diff --git a/models/base/google/streaming/information_schema_streaming_timeline.sql b/models/base/google/streaming/information_schema_streaming_timeline.sql index b00cc93..4a75a73 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline.sql +++ b/models/base/google/streaming/information_schema_streaming_timeline.sql @@ -1,5 +1,14 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming -#} -SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes +SELECT +start_timestamp, +project_id, +project_number, +dataset_id, +table_id, +error_code, +total_requests, +total_rows, +total_input_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE` diff --git a/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql b/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql index 262ef27..5244069 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql +++ b/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql @@ -1,5 +1,14 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming-by-folder -#} -SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes +SELECT +start_timestamp, +project_id, +project_number, +dataset_id, +table_id, +error_code, +total_requests, +total_rows, +total_input_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_FOLDER` diff --git a/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql b/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql index 118cec6..7dc123d 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql +++ b/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql @@ -1,5 +1,14 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming-by-organization -#} -SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes +SELECT +start_timestamp, +project_id, +project_number, +dataset_id, +table_id, +error_code, +total_requests, +total_rows, +total_input_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_ORGANIZATION` diff --git a/models/base/google/tables/information_schema_column_field_paths.sql b/models/base/google/tables/information_schema_column_field_paths.sql index 77c37a0..ce03f16 100644 --- a/models/base/google/tables/information_schema_column_field_paths.sql +++ b/models/base/google/tables/information_schema_column_field_paths.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-column-field-paths -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.COLUMN_FIELD_PATHS view, you need the following Identity and Access Management (IAM) permissions: @@ -13,5 +13,14 @@ roles/bigquery.metadataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT table_catalog, table_schema, table_name, column_name, field_path, data_type, description, collation_name, rounding_mode +SELECT +table_catalog, +table_schema, +table_name, +column_name, +field_path, +data_type, +description, +collation_name, +rounding_mode FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMN_FIELD_PATHS` diff --git a/models/base/google/tables/information_schema_columns.sql b/models/base/google/tables/information_schema_columns.sql index d3a4a82..b1c6b29 100644 --- a/models/base/google/tables/information_schema_columns.sql +++ b/models/base/google/tables/information_schema_columns.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-columns -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.COLUMNS view, you need the following Identity and Access Management (IAM) permissions: @@ -13,5 +13,23 @@ roles/bigquery.metadataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT table_catalog, table_schema, table_name, column_name, ordinal_position, is_nullable, data_type, is_generated, generation_expression, is_stored, is_hidden, is_updatable, is_system_defined, is_partitioning_column, clustering_ordinal_position, collation_name, column_default, rounding_mode +SELECT +table_catalog, +table_schema, +table_name, +column_name, +ordinal_position, +is_nullable, +data_type, +is_generated, +generation_expression, +is_stored, +is_hidden, +is_updatable, +is_system_defined, +is_partitioning_column, +clustering_ordinal_position, +collation_name, +column_default, +rounding_mode FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMNS` diff --git a/models/base/google/tables/information_schema_constraint_column_usage.sql b/models/base/google/tables/information_schema_constraint_column_usage.sql index 8a78978..a7f230e 100644 --- a/models/base/google/tables/information_schema_constraint_column_usage.sql +++ b/models/base/google/tables/information_schema_constraint_column_usage.sql @@ -1,32 +1,44 @@ +{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-constraint-column-usage -#} - {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-constraint-column-usage -#} - - - {% set preflight_sql -%} - SELECT - CONCAT('`', CATALOG_NAME, '`.`', SCHEMA_NAME, '`') AS SCHEMA_NAME - FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` - {%- endset %} - {% set results = run_query(preflight_sql) %} - {% set dataset_list = results | map(attribute='SCHEMA_NAME') | list %} - {%- if dataset_list | length == 0 -%} - {{ log("No datasets found in the project list", info=False) }} - {%- endif -%} - - WITH base AS ( - {%- if dataset_list | length == 0 -%} - SELECT CAST(NULL AS STRING) AS table_catalog, CAST(NULL AS STRING) AS table_schema, CAST(NULL AS STRING) AS table_name, CAST(NULL AS STRING) AS column_name, CAST(NULL AS STRING) AS constraint_catalog, CAST(NULL AS STRING) AS constraint_schema, CAST(NULL AS STRING) AS constraint_name - LIMIT 0 - {%- else %} - {% for dataset in dataset_list -%} - SELECT table_catalog, table_schema, table_name, column_name, constraint_catalog, constraint_schema, constraint_name - FROM {{ dataset | trim }}.`INFORMATION_SCHEMA`.`PARTITIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} - {%- endif -%} - ) - SELECT - table_catalog, table_schema, table_name, column_name, constraint_catalog, constraint_schema, constraint_name, - FROM - base - \ No newline at end of file + +{% set preflight_sql -%} +SELECT +CONCAT('`', CATALOG_NAME, '`.`', SCHEMA_NAME, '`') AS SCHEMA_NAME +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` +{%- endset %} +{% set results = run_query(preflight_sql) %} +{% set dataset_list = results | map(attribute='SCHEMA_NAME') | list %} +{%- if dataset_list | length == 0 -%} +{{ log("No datasets found in the project list", info=False) }} +{%- endif -%} + +WITH base AS ( +{%- if dataset_list | length == 0 -%} + SELECT CAST(NULL AS STRING) AS table_catalog, CAST(NULL AS STRING) AS table_schema, CAST(NULL AS STRING) AS table_name, CAST(NULL AS STRING) AS column_name, CAST(NULL AS STRING) AS constraint_catalog, CAST(NULL AS STRING) AS constraint_schema, CAST(NULL AS STRING) AS constraint_name + LIMIT 0 +{%- else %} +{% for dataset in dataset_list -%} + SELECT + table_catalog, +table_schema, +table_name, +column_name, +constraint_catalog, +constraint_schema, +constraint_name + FROM {{ dataset | trim }}.`INFORMATION_SCHEMA`.`PARTITIONS` +{% if not loop.last %}UNION ALL{% endif %} +{% endfor %} +{%- endif -%} +) + +SELECT +table_catalog, +table_schema, +table_name, +column_name, +constraint_catalog, +constraint_schema, +constraint_name, +FROM +base diff --git a/models/base/google/tables/information_schema_key_column_usage.sql b/models/base/google/tables/information_schema_key_column_usage.sql index 8688ba0..e707d9c 100644 --- a/models/base/google/tables/information_schema_key_column_usage.sql +++ b/models/base/google/tables/information_schema_key_column_usage.sql @@ -1,32 +1,48 @@ +{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-key-column-usage -#} - {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-key-column-usage -#} - - - {% set preflight_sql -%} - SELECT - CONCAT('`', CATALOG_NAME, '`.`', SCHEMA_NAME, '`') AS SCHEMA_NAME - FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` - {%- endset %} - {% set results = run_query(preflight_sql) %} - {% set dataset_list = results | map(attribute='SCHEMA_NAME') | list %} - {%- if dataset_list | length == 0 -%} - {{ log("No datasets found in the project list", info=False) }} - {%- endif -%} - - WITH base AS ( - {%- if dataset_list | length == 0 -%} - SELECT CAST(NULL AS STRING) AS constraint_catalog, CAST(NULL AS STRING) AS constraint_schema, CAST(NULL AS STRING) AS constraint_name, CAST(NULL AS STRING) AS table_catalog, CAST(NULL AS STRING) AS table_schema, CAST(NULL AS STRING) AS table_name, CAST(NULL AS STRING) AS column_name, CAST(NULL AS INT64) AS ordinal_position, CAST(NULL AS INT64) AS position_in_unique_constraint - LIMIT 0 - {%- else %} - {% for dataset in dataset_list -%} - SELECT constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, column_name, ordinal_position, position_in_unique_constraint - FROM {{ dataset | trim }}.`INFORMATION_SCHEMA`.`PARTITIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} - {%- endif -%} - ) - SELECT - constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, column_name, ordinal_position, position_in_unique_constraint, - FROM - base - \ No newline at end of file + +{% set preflight_sql -%} +SELECT +CONCAT('`', CATALOG_NAME, '`.`', SCHEMA_NAME, '`') AS SCHEMA_NAME +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` +{%- endset %} +{% set results = run_query(preflight_sql) %} +{% set dataset_list = results | map(attribute='SCHEMA_NAME') | list %} +{%- if dataset_list | length == 0 -%} +{{ log("No datasets found in the project list", info=False) }} +{%- endif -%} + +WITH base AS ( +{%- if dataset_list | length == 0 -%} + SELECT CAST(NULL AS STRING) AS constraint_catalog, CAST(NULL AS STRING) AS constraint_schema, CAST(NULL AS STRING) AS constraint_name, CAST(NULL AS STRING) AS table_catalog, CAST(NULL AS STRING) AS table_schema, CAST(NULL AS STRING) AS table_name, CAST(NULL AS STRING) AS column_name, CAST(NULL AS INT64) AS ordinal_position, CAST(NULL AS INT64) AS position_in_unique_constraint + LIMIT 0 +{%- else %} +{% for dataset in dataset_list -%} + SELECT + constraint_catalog, +constraint_schema, +constraint_name, +table_catalog, +table_schema, +table_name, +column_name, +ordinal_position, +position_in_unique_constraint + FROM {{ dataset | trim }}.`INFORMATION_SCHEMA`.`PARTITIONS` +{% if not loop.last %}UNION ALL{% endif %} +{% endfor %} +{%- endif -%} +) + +SELECT +constraint_catalog, +constraint_schema, +constraint_name, +table_catalog, +table_schema, +table_name, +column_name, +ordinal_position, +position_in_unique_constraint, +FROM +base diff --git a/models/base/google/tables/information_schema_partitions.sql b/models/base/google/tables/information_schema_partitions.sql index d02b032..c72d0a9 100644 --- a/models/base/google/tables/information_schema_partitions.sql +++ b/models/base/google/tables/information_schema_partitions.sql @@ -1,6 +1,5 @@ - - {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-partitions -#} - {# Required role/permissions: To query the INFORMATION_SCHEMA.PARTITIONS view, you need the following +{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-partitions -#} +{# Required role/permissions: To query the INFORMATION_SCHEMA.PARTITIONS view, you need the following Identity and Access Management (IAM) permissions: bigquery.tables.get bigquery.tables.getData @@ -13,32 +12,47 @@ roles/bigquery.dataViewer For more information about BigQuery permissions, see Access control with IAM. -#} - - {% set preflight_sql -%} - SELECT - CONCAT('`', CATALOG_NAME, '`.`', SCHEMA_NAME, '`') AS SCHEMA_NAME - FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` - {%- endset %} - {% set results = run_query(preflight_sql) %} - {% set dataset_list = results | map(attribute='SCHEMA_NAME') | list %} - {%- if dataset_list | length == 0 -%} - {{ log("No datasets found in the project list", info=False) }} - {%- endif -%} - - WITH base AS ( - {%- if dataset_list | length == 0 -%} - SELECT CAST(NULL AS STRING) AS table_catalog, CAST(NULL AS STRING) AS table_schema, CAST(NULL AS STRING) AS table_name, CAST(NULL AS STRING) AS partition_id, CAST(NULL AS INTEGER) AS total_rows, CAST(NULL AS INTEGER) AS total_logical_bytes, CAST(NULL AS TIMESTAMP) AS last_modified_time, CAST(NULL AS STRING) AS storage_tier - LIMIT 0 - {%- else %} - {% for dataset in dataset_list -%} - SELECT table_catalog, table_schema, table_name, partition_id, total_rows, total_logical_bytes, last_modified_time, storage_tier - FROM {{ dataset | trim }}.`INFORMATION_SCHEMA`.`PARTITIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} - {%- endif -%} - ) - SELECT - table_catalog, table_schema, table_name, partition_id, total_rows, total_logical_bytes, last_modified_time, storage_tier, - FROM - base - \ No newline at end of file + +{% set preflight_sql -%} +SELECT +CONCAT('`', CATALOG_NAME, '`.`', SCHEMA_NAME, '`') AS SCHEMA_NAME +FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` +{%- endset %} +{% set results = run_query(preflight_sql) %} +{% set dataset_list = results | map(attribute='SCHEMA_NAME') | list %} +{%- if dataset_list | length == 0 -%} +{{ log("No datasets found in the project list", info=False) }} +{%- endif -%} + +WITH base AS ( +{%- if dataset_list | length == 0 -%} + SELECT CAST(NULL AS STRING) AS table_catalog, CAST(NULL AS STRING) AS table_schema, CAST(NULL AS STRING) AS table_name, CAST(NULL AS STRING) AS partition_id, CAST(NULL AS INTEGER) AS total_rows, CAST(NULL AS INTEGER) AS total_logical_bytes, CAST(NULL AS TIMESTAMP) AS last_modified_time, CAST(NULL AS STRING) AS storage_tier + LIMIT 0 +{%- else %} +{% for dataset in dataset_list -%} + SELECT + table_catalog, +table_schema, +table_name, +partition_id, +total_rows, +total_logical_bytes, +last_modified_time, +storage_tier + FROM {{ dataset | trim }}.`INFORMATION_SCHEMA`.`PARTITIONS` +{% if not loop.last %}UNION ALL{% endif %} +{% endfor %} +{%- endif -%} +) + +SELECT +table_catalog, +table_schema, +table_name, +partition_id, +total_rows, +total_logical_bytes, +last_modified_time, +storage_tier, +FROM +base diff --git a/models/base/google/tables/information_schema_table_constraints.sql b/models/base/google/tables/information_schema_table_constraints.sql index c180b75..b092af4 100644 --- a/models/base/google/tables/information_schema_table_constraints.sql +++ b/models/base/google/tables/information_schema_table_constraints.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-constraints -#} {# Required role/permissions: You need the following Identity and Access Management (IAM) permissions: @@ -16,5 +16,15 @@ excess permissions.For more information about IAM roles and permissions in BigQuery, see Predefined roles and permissions. -#} -SELECT constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, constraint_type, is_deferrable, initially_deferred, enforced +SELECT +constraint_catalog, +constraint_schema, +constraint_name, +table_catalog, +table_schema, +table_name, +constraint_type, +is_deferrable, +initially_deferred, +enforced FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_CONSTRAINTS` diff --git a/models/base/google/tables/information_schema_table_options.sql b/models/base/google/tables/information_schema_table_options.sql index d6e6a69..58a110c 100644 --- a/models/base/google/tables/information_schema_table_options.sql +++ b/models/base/google/tables/information_schema_table_options.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-options -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_OPTIONS view, you need the following Identity and Access Management (IAM) permissions: @@ -14,5 +14,11 @@ roles/bigquery.metadataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT table_catalog, table_schema, table_name, option_name, option_type, option_value +SELECT +table_catalog, +table_schema, +table_name, +option_name, +option_type, +option_value FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_OPTIONS` diff --git a/models/base/google/tables/information_schema_table_snapshots.sql b/models/base/google/tables/information_schema_table_snapshots.sql index 978392c..3ff7515 100644 --- a/models/base/google/tables/information_schema_table_snapshots.sql +++ b/models/base/google/tables/information_schema_table_snapshots.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-snapshots -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_SNAPSHOTS view, you need the bigquery.tables.list Identity and Access Management (IAM) permission for the dataset. @@ -6,5 +6,12 @@ The roles/bigquery.metadataViewer predefined role includes the required permission.For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT table_catalog, table_schema, table_name, base_table_catalog, base_table_schema, base_table_name, snapshot_time +SELECT +table_catalog, +table_schema, +table_name, +base_table_catalog, +base_table_schema, +base_table_name, +snapshot_time FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_SNAPSHOTS` diff --git a/models/base/google/tables/information_schema_table_storage.sql b/models/base/google/tables/information_schema_table_storage.sql index e43a455..63bf4aa 100644 --- a/models/base/google/tables/information_schema_table_storage.sql +++ b/models/base/google/tables/information_schema_table_storage.sql @@ -1,5 +1,26 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage -#} -SELECT project_id, project_number, table_catalog, table_schema, table_name, creation_time, total_rows, total_partitions, total_logical_bytes, active_logical_bytes, long_term_logical_bytes, current_physical_bytes, total_physical_bytes, active_physical_bytes, long_term_physical_bytes, time_travel_physical_bytes, storage_last_modified_time, deleted, table_type, fail_safe_physical_bytes, last_metadata_index_refresh_time +SELECT +project_id, +project_number, +table_catalog, +table_schema, +table_name, +creation_time, +total_rows, +total_partitions, +total_logical_bytes, +active_logical_bytes, +long_term_logical_bytes, +current_physical_bytes, +total_physical_bytes, +active_physical_bytes, +long_term_physical_bytes, +time_travel_physical_bytes, +storage_last_modified_time, +deleted, +table_type, +fail_safe_physical_bytes, +last_metadata_index_refresh_time FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE` diff --git a/models/base/google/tables/information_schema_table_storage_by_organization.sql b/models/base/google/tables/information_schema_table_storage_by_organization.sql index e0549d0..7459eac 100644 --- a/models/base/google/tables/information_schema_table_storage_by_organization.sql +++ b/models/base/google/tables/information_schema_table_storage_by_organization.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-by-organization -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_BY_ORGANIZATION view, you need the following Identity and Access Management (IAM) permissions for your organization: @@ -14,5 +14,26 @@ This schema view is only available to users with defined Google Cloud organizations.For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT project_id, project_number, table_catalog, table_schema, table_name, creation_time, total_rows, total_partitions, total_logical_bytes, active_logical_bytes, long_term_logical_bytes, current_physical_bytes, total_physical_bytes, active_physical_bytes, long_term_physical_bytes, time_travel_physical_bytes, storage_last_modified_time, deleted, table_type, fail_safe_physical_bytes, last_metadata_index_refresh_time +SELECT +project_id, +project_number, +table_catalog, +table_schema, +table_name, +creation_time, +total_rows, +total_partitions, +total_logical_bytes, +active_logical_bytes, +long_term_logical_bytes, +current_physical_bytes, +total_physical_bytes, +active_physical_bytes, +long_term_physical_bytes, +time_travel_physical_bytes, +storage_last_modified_time, +deleted, +table_type, +fail_safe_physical_bytes, +last_metadata_index_refresh_time FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_BY_ORGANIZATION` diff --git a/models/base/google/tables/information_schema_table_storage_usage_timeline.sql b/models/base/google/tables/information_schema_table_storage_usage_timeline.sql index 328397a..809b029 100644 --- a/models/base/google/tables/information_schema_table_storage_usage_timeline.sql +++ b/models/base/google/tables/information_schema_table_storage_usage_timeline.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-usage -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_USAGE_TIMELINE view, you need the following Identity and Access Management (IAM) permissions: @@ -13,5 +13,16 @@ roles/bigquery.admin For queries with a region qualifier, you must have permissions for the project.For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT project_id, table_catalog, project_number, table_schema, table_name, billable_total_logical_usage, billable_active_logical_usage, billable_long_term_logical_usage, billable_total_physical_usage, billable_active_physical_usage, billable_long_term_physical_usage +SELECT +project_id, +table_catalog, +project_number, +table_schema, +table_name, +billable_total_logical_usage, +billable_active_logical_usage, +billable_long_term_logical_usage, +billable_total_physical_usage, +billable_active_physical_usage, +billable_long_term_physical_usage FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE` diff --git a/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql b/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql index aa744b5..83555bd 100644 --- a/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql +++ b/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-usage-by-organization -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_USAGE_TIMELINE_BY_ORGANIZATION view, you need the following Identity and Access Management (IAM) permissions for your organization: @@ -14,5 +14,16 @@ This schema view is only available to users with defined Google Cloud organizations.For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT project_id, table_catalog, project_number, table_schema, table_name, billable_total_logical_usage, billable_active_logical_usage, billable_long_term_logical_usage, billable_total_physical_usage, billable_active_physical_usage, billable_long_term_physical_usage +SELECT +project_id, +table_catalog, +project_number, +table_schema, +table_name, +billable_total_logical_usage, +billable_active_logical_usage, +billable_long_term_logical_usage, +billable_total_physical_usage, +billable_active_physical_usage, +billable_long_term_physical_usage FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE_BY_ORGANIZATION` diff --git a/models/base/google/tables/information_schema_tables.sql b/models/base/google/tables/information_schema_tables.sql index 4246d67..bf837bd 100644 --- a/models/base/google/tables/information_schema_tables.sql +++ b/models/base/google/tables/information_schema_tables.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-tables -#} {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLES view, you need the following Identity and Access Management (IAM) permissions: @@ -14,5 +14,25 @@ roles/bigquery.metadataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT table_catalog, table_schema, table_name, table_type, is_insertable_into, is_typed, is_change_history_enabled, creation_time, base_table_catalog, base_table_schema, base_table_name, snapshot_time_ms, replica_source_catalog, replica_source_schema, replica_source_name, replication_status, replication_error, ddl, default_collation_name, upsert_stream_apply_watermark +SELECT +table_catalog, +table_schema, +table_name, +table_type, +is_insertable_into, +is_typed, +is_change_history_enabled, +creation_time, +base_table_catalog, +base_table_schema, +base_table_name, +snapshot_time_ms, +replica_source_catalog, +replica_source_schema, +replica_source_name, +replication_status, +replication_error, +ddl, +default_collation_name, +upsert_stream_apply_watermark FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLES` diff --git a/models/base/google/vector_indexes/information_schema_vector_index_columns.sql b/models/base/google/vector_indexes/information_schema_vector_index_columns.sql index 15bbc57..4c1845b 100644 --- a/models/base/google/vector_indexes/information_schema_vector_index_columns.sql +++ b/models/base/google/vector_indexes/information_schema_vector_index_columns.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-index-columns -#} {# Required role/permissions: To see vector index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) @@ -13,5 +13,11 @@ roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT index_catalog, index_schema, table_name, index_name, index_column_name, index_field_path +SELECT +index_catalog, +index_schema, +table_name, +index_name, +index_column_name, +index_field_path FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_COLUMNS` diff --git a/models/base/google/vector_indexes/information_schema_vector_index_options.sql b/models/base/google/vector_indexes/information_schema_vector_index_options.sql index dbf38ab..fe6e582 100644 --- a/models/base/google/vector_indexes/information_schema_vector_index_options.sql +++ b/models/base/google/vector_indexes/information_schema_vector_index_options.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-index-options -#} {# Required role/permissions: To see vector index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) @@ -13,5 +13,12 @@ roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT index_catalog, index_schema, table_name, index_name, option_name, option_type, option_value +SELECT +index_catalog, +index_schema, +table_name, +index_name, +option_name, +option_type, +option_value FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_OPTIONS` diff --git a/models/base/google/vector_indexes/information_schema_vector_indexes.sql b/models/base/google/vector_indexes/information_schema_vector_indexes.sql index 1f11669..0a53070 100644 --- a/models/base/google/vector_indexes/information_schema_vector_indexes.sql +++ b/models/base/google/vector_indexes/information_schema_vector_indexes.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-indexes -#} {# Required role/permissions: To see vector index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) @@ -13,5 +13,20 @@ roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT index_catalog, index_schema, table_name, index_name, index_status, creation_time, last_modification_time, last_refresh_time, disable_time, disable_reason, ddl, coverage_percentage, unindexed_row_count, total_logical_bytes, total_storage_bytes +SELECT +index_catalog, +index_schema, +table_name, +index_name, +index_status, +creation_time, +last_modification_time, +last_refresh_time, +disable_time, +disable_reason, +ddl, +coverage_percentage, +unindexed_row_count, +total_logical_bytes, +total_storage_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEXES` diff --git a/models/base/google/views/information_schema_materialized_views.sql b/models/base/google/views/information_schema_materialized_views.sql index c4d1ca0..f2850ff 100644 --- a/models/base/google/views/information_schema_materialized_views.sql +++ b/models/base/google/views/information_schema_materialized_views.sql @@ -1,6 +1,6 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-materialized-views -#} -{# Required role/permissions: +{# Required role/permissions: To get the permissions that you need to query the INFORMATION_SCHEMA.MATERIALIZED_VIEWS view, @@ -28,7 +28,7 @@ Required permissions The following permissions are required to query the INFORMATION_SCHEMA.MATERIALIZED_VIEWS view: - bigquery.tables.get + bigquery.tables.get bigquery.tables.list @@ -41,5 +41,11 @@ The following permissions are required to query the INFORMATION_SCHEMA.MATERIALI other predefined roles. Access control with IAM -#} -SELECT table_catalog, table_schema, table_name, last_refresh_time, refresh_watermark, last_refresh_status +SELECT +table_catalog, +table_schema, +table_name, +last_refresh_time, +refresh_watermark, +last_refresh_status FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`MATERIALIZED_VIEWS` diff --git a/models/base/google/views/information_schema_views.sql b/models/base/google/views/information_schema_views.sql index f8ca33b..7ec97f2 100644 --- a/models/base/google/views/information_schema_views.sql +++ b/models/base/google/views/information_schema_views.sql @@ -1,4 +1,4 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-views -#} {# Required role/permissions: To get view metadata, you need the following Identity and Access Management (IAM) permissions: @@ -13,5 +13,11 @@ roles/bigquery.dataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -SELECT table_catalog, table_schema, table_name, view_definition, check_option, use_standard_sql +SELECT +table_catalog, +table_schema, +table_name, +view_definition, +check_option, +use_standard_sql FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VIEWS` diff --git a/models/base/google/write_api/information_schema_write_api_timeline.sql b/models/base/google/write_api/information_schema_write_api_timeline.sql index 7624281..fa92812 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline.sql +++ b/models/base/google/write_api/information_schema_write_api_timeline.sql @@ -1,5 +1,15 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api -#} -SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes +SELECT +start_timestamp, +project_id, +project_number, +dataset_id, +table_id, +stream_type, +error_code, +total_requests, +total_rows, +total_input_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE` diff --git a/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql b/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql index ac89678..dac5d14 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql +++ b/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql @@ -1,5 +1,15 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api-by-folder -#} -SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes +SELECT +start_timestamp, +project_id, +project_number, +dataset_id, +table_id, +stream_type, +error_code, +total_requests, +total_rows, +total_input_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_FOLDER` diff --git a/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql b/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql index f7afdfb..818315a 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql +++ b/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql @@ -1,5 +1,15 @@ -{{ config(materialization=dbt_bigquery_monitoring_materialization()) }} +{{ config(materialized=dbt_bigquery_monitoring_materialization()) }} {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api-by-organization -#} -SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes +SELECT +start_timestamp, +project_id, +project_number, +dataset_id, +table_id, +stream_type, +error_code, +total_requests, +total_rows, +total_input_bytes FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_ORGANIZATION` diff --git a/models/base/jobs_from_audit_logs.sql b/models/base/jobs_from_audit_logs.sql index c0b7021..5869538 100644 --- a/models/base/jobs_from_audit_logs.sql +++ b/models/base/jobs_from_audit_logs.sql @@ -5,8 +5,6 @@ ) }} SELECT - -- bi_engine_statistics is not available in the audit logs, so we default to NULL - CAST(NULL AS STRUCT>, acceleration_mode STRING>) AS bi_engine_statistics, CAST(JSON_VALUE(protopayload_auditlog.metadataJson, '$.jobChange.job.jobStats.queryStats.cacheHit') AS BOOL) AS cache_hit, TIMESTAMP(JSON_VALUE(protopayload_auditlog.metadataJson, From 036fea22f9f529993c0f48c5cd30c67fabd923a5 Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Mon, 13 Jan 2025 00:31:49 +0100 Subject: [PATCH 11/12] Update the doc to reflect the new approach --- docs/configuration/configuration.md | 8 ++++++++ docs/configuration/package-settings.md | 1 + 2 files changed, 9 insertions(+) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 6e0045e..dfaec7b 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -45,6 +45,7 @@ vars: ### Project mode +Project mode is useful when you have multiple GCP projects or you want to store the dbt-bigquery-monitoring models in a project different from the one used for execution. To enable the "project mode", you'll need to define explicitly one mandatory setting to set in the `dbt_project.yml` file: ```yml @@ -53,6 +54,13 @@ vars: input_gcp_projects: [ 'my-gcp-project', 'my-gcp-project-2' ] ``` +:::warning + +When using the "project mode", the package will create intermediate tables to avoid issues from BigQuery when too many projects are used. +That process is done only on tables that are project related. The package leverages a custom materialiation (`project_by_project_table`) designed specifically for that need that can found in the `macros` folder. + +::: + ## Add metadata to queries (Recommended but optional) To enhance your query metadata with dbt model information, the package provides a dedicated macro that leverage "dbt query comments" (the header set at the top of each query) diff --git a/docs/configuration/package-settings.md b/docs/configuration/package-settings.md index 586114c..3c94341 100644 --- a/docs/configuration/package-settings.md +++ b/docs/configuration/package-settings.md @@ -41,6 +41,7 @@ These settings are used to configure how dbt will run and materialize the models | `output_limit_size` | `DBT_BQ_MONITORING_OUTPUT_LIMIT_SIZE` | Limit size to use for the models | `1000` | | `output_partition_expiration_days` | `DBT_BQ_MONITORING_OUTPUT_LIMIT_SIZE` | Default table expiration in days for incremental models | `365` days | | `use_copy_partitions` | `DBT_BQ_MONITORING_USE_COPY_PARTITIONS` | Whether to use copy partitions or not | `true` | +| `google_information_schema_model_materialization` | `DBT_BQ_MONITORING_GOOGLE_INFORMATION_SCHEMA_MODELS_MATERIALIZATION` | Whether to use a specific materialization for information schema models. Note that it doesn't work in project mode as it will materialize intermediate tables to avoid issues from BQ when too many projects are used. | `ephemeral` | ### GCP Billing export configuration From ed9e7b63f1c2e885dbe5012f4d521e5bc6f9f15e Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Mon, 13 Jan 2025 00:32:58 +0100 Subject: [PATCH 12/12] fix contributing.md content --- CONTRIBUTING.md | 96 ++----------------------------------------------- 1 file changed, 2 insertions(+), 94 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 525b151..4128fa6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,96 +1,4 @@ # Contributing to dbt-bigquery-monitoring -## Install setup - -You're free to use the environment management tools you prefer but if you're familiar with those, you can use the following: - -- pipx (to isolate the global tools from your local environment) -- tox (to run the tests) -- pre-commit (to run the linter) -- SQLFluff (to lint SQL) -- changie (to generate CHANGELOG entries) - -### tool setup guide - -To install pipx: - -```bash -pip install pipx -pipx ensurepath -``` - -Then you'll be able to install tox, pre-commit and sqlfluff with pipx: - -```bash -pipx install tox -pipx install pre-commit -pipx install sqlfluff -``` - -To install changie, there are few options depending on your OS. -See the [installation guide](https://changie.dev/guide/installation/) for more details. - -To configure pre-commit hooks: - -```bash -pre-commit install -``` - -To configure your dbt profile, run following command and follow the prompts: - -```bash -dbt init -``` - -## Development workflow - -- Fork the repo -- Create a branch from `main` -- Make your changes -- Run `tox` to run the tests -- Create your changelog entry with `changie new` (don't edit directly the CHANGELOG.md) -- Commit your changes (it will run the linter through pre-commit) -- Push your branch and open a PR on the repository - -## Adding a CHANGELOG Entry - -We use changie to generate CHANGELOG entries. Note: Do not edit the CHANGELOG.md directly. Your modifications will be lost. - -Follow the steps to [install changie](https://changie.dev/guide/installation/) for your system. - -Once changie is installed and your PR is created, simply run `changie new` and changie will walk you through the process of creating a changelog entry. Commit the file that's created and your changelog entry is complete! - -### SQLFluff - -We use SQLFluff to keep SQL style consistent. By installing `pre-commit` per the initial setup guide above, SQLFluff will run automatically when you make a commit locally. A GitHub action automatically tests pull requests and adds annotations where there are failures. - -Lint all models in the /models directory: -```bash -tox -e lint_all -``` - -Fix all models in the /models directory: -```bash -tox -e fix_all -``` - -Lint (or subsitute lint to fix) a specific model: -```bash -tox -e lint -- models/path/to/model.sql -``` - -Lint (or subsitute lint to fix) a specific directory: -```bash -tox -e lint -- models/path/to/directory -``` - -#### Rules - -Enforced rules are defined within `tox.ini`. To view the full list of available rules and their configuration, see the [SQLFluff documentation](https://docs.sqlfluff.com/en/stable/rules.html). - -## Generation of dbt base google models - -dbt base google models are generated in another dedicated project hosted in: -https://github.com/bqbooster/dbt-bigquery-monitoring-parser - -It was separated to ensure that users don't install the parser (and tests) when they install the dbt package. +See related documentation page: +https://bqbooster.github.io/dbt-bigquery-monitoring/contributing