diff --git a/.changes/v0.5.0.md b/.changes/v0.5.0.md new file mode 100644 index 0000000..67670dd --- /dev/null +++ b/.changes/v0.5.0.md @@ -0,0 +1,10 @@ +## dbt-bigquery-monitoring v0.5.0 - June 05, 2024 + +### Features + +- Add new models for storage monitoring ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0)) +- Add SQLMesh support ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0)) + +### Contributors +- [@Kayrnt](https://github.com/Kayrnt) ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0), [#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0)) + diff --git a/.changes/v0.5.1.md b/.changes/v0.5.1.md new file mode 100644 index 0000000..462c5db --- /dev/null +++ b/.changes/v0.5.1.md @@ -0,0 +1,9 @@ +## dbt-bigquery-monitoring v0.5.1 - July 30, 2024 + +### Fixes + +- Fix YML model name misaligned with information schema dbt models ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0)) + +### Contributors +- [@Kayrnt](https://github.com/Kayrnt) ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0)) + diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a52a0f..ad97d9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-bigquery-monitoring v0.5.0 - June 05, 2024 + +### Features + +- Add new models for storage monitoring ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0)) +- Add SQLMesh support ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0)) + +### Contributors +- [@Kayrnt](https://github.com/Kayrnt) ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0), [#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0)) + + ## dbt-bigquery-monitoring v0.4.0 - May 18, 2024 ### Features diff --git a/README.md b/README.md index 7b2c1bf..468d393 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Add the following to your `packages.yml` file: ```yml packages: - package: bqbooster/dbt_bigquery_monitoring - version: 0.4.0 + version: 0.5.0 ``` ### Configure the package diff --git a/config.py b/config.py new file mode 100644 index 0000000..20ae743 --- /dev/null +++ b/config.py @@ -0,0 +1,10 @@ +# This file is used to load the sqlmesh configuration +from pathlib import Path + +from sqlmesh.core.config import DuckDBConnectionConfig +from sqlmesh.dbt.loader import sqlmesh_config + +config = sqlmesh_config( + Path(__file__).parent, + state_connection=DuckDBConnectionConfig(), +) diff --git a/dbt_project.yml b/dbt_project.yml index e6fb931..785e825 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: "dbt_bigquery_monitoring" -version: "0.3.0" +version: "0.5.1" require-dbt-version: [">=1.3.0", "<2.0.0"] config-version: 2 @@ -12,6 +12,7 @@ clean-targets: - dbt_packages models: + +start: Jan 1 2017 dbt_bigquery_monitoring: +tags: - "dbt-bigquery-monitoring" @@ -38,6 +39,7 @@ vars: # Capacity compute (analysis) pricing hourly_slot_price: "{{ env_var('DBT_BQ_MONITORING_HOURLY_SLOT_PRICE', 0.04) }}" # https://cloud.google.com/bigquery/pricing#storage + prefer_physical_pricing_model: "{{ env_var('DBT_BQ_MONITORING_PREFER_PHYSICAL_PRICING_MODEL', true) }}" active_logical_storage_gb_price: "{{ env_var('DBT_BQ_MONITORING_ACTIVE_LOGICAL_STORAGE_GB_PRICE', 0.02) }}" long_term_logical_storage_gb_price: "{{ env_var('DBT_BQ_MONITORING_LONG_TERM_LOGICAL_STORAGE_GB_PRICE', 0.01) }}" active_physical_storage_gb_price: "{{ env_var('DBT_BQ_MONITORING_ACTIVE_PHYSICAL_STORAGE_GB_PRICE', 0.04) }}" diff --git a/documentation_parser/Makefile b/documentation_parser/Makefile index 3c6dc36..6c3711e 100644 --- a/documentation_parser/Makefile +++ b/documentation_parser/Makefile @@ -14,3 +14,8 @@ setup_environment: run: python documentation_parser.py + +#test using pytest +test: + pytest test_documentation_parser.py + diff --git a/documentation_parser/config.py b/documentation_parser/config.py index e6c8add..113e436 100644 --- a/documentation_parser/config.py +++ b/documentation_parser/config.py @@ -60,34 +60,39 @@ "jobs": { "dir": "jobs", "url": "https://cloud.google.com/bigquery/docs/information-schema-jobs", + "exclude_columns": ["query_info.resource_warning", "query_info.query_hashes.normalized_literals", "query_info.performance_insights", "query_info.optimization_details", "folder_numbers"], }, "jobs_by_project": { "dir": "jobs", "url": "https://cloud.google.com/bigquery/docs/information-schema-jobs", "override_table_name": "JOBS_BY_PROJECT", + "exclude_columns": ["query_info.resource_warning", "query_info.query_hashes.normalized_literals", "query_info.performance_insights", "query_info.optimization_details", "folder_numbers"], }, "jobs_by_user": { "dir": "jobs", "url": "https://cloud.google.com/bigquery/docs/information-schema-jobs-by-user", + "exclude_columns": ["query_info.resource_warning", "query_info.query_hashes.normalized_literals", "query_info.performance_insights", "query_info.optimization_details", "folder_numbers"], }, "jobs_by_folder": { "dir": "jobs", "url": "https://cloud.google.com/bigquery/docs/information-schema-jobs-by-folder", + "exclude_columns": ["query_info.resource_warning", "query_info.query_hashes.normalized_literals", "query_info.performance_insights", "query_info.optimization_details", "folder_numbers"], }, "jobs_by_organization": { "dir": "jobs", "url": "https://cloud.google.com/bigquery/docs/information-schema-jobs-by-organization", + "exclude_columns": ["query_info.resource_warning", "query_info.query_hashes.normalized_literals", "query_info.performance_insights", "query_info.optimization_details", "folder_numbers"], }, # jobs timeline "jobs_timeline": { "dir": "jobs_timeline", "url": "https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline", - "exclude_columns": ["folder_numbers"], + "exclude_columns": ["query_info.resource_warning", "query_info.query_hashes.normalized_literals", "query_info.performance_insights", "query_info.optimization_details", "folder_numbers"], }, "jobs_timeline_by_user": { "dir": "jobs_timeline", "url": "https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-user", - "exclude_columns": ["folder_numbers"], + "exclude_columns": ["query_info.resource_warning", "query_info.query_hashes.normalized_literals", "query_info.performance_insights", "query_info.optimization_details", "folder_numbers"], }, "jobs_timeline_by_folder": { "dir": "jobs_timeline", diff --git a/documentation_parser/documentation_parser.py b/documentation_parser/documentation_parser.py index 8730ebe..ab3ba84 100644 --- a/documentation_parser/documentation_parser.py +++ b/documentation_parser/documentation_parser.py @@ -83,6 +83,42 @@ def update_column_list(input_columns: List[dict], exclude_columns: List[str]): return columns +def generate_sql(url: str, column_names: List[str], table_name: str, required_role_str: str): + # Prepare the column names as a comma-separated string + columns_str = ", ".join(column_names) + + # Prepare the base SQL string for the project list + project_sql = f""" + SELECT {columns_str} + FROM `{{{{ project | trim }}}}`.`region-{{{{ var('bq_region') }}}}`.`INFORMATION_SCHEMA`.`{table_name}` + """ + + # Prepare the base SQL string for when there's no project list + no_project_sql = f""" + SELECT {columns_str} + FROM `region-{{{{ var('bq_region') }}}}`.`INFORMATION_SCHEMA`.`{table_name}` + """ + + # Combine everything into the final SQL string + sql = f""" + {{# More details about base table in {url} -#}} + {required_role_str} + WITH base AS ( + {{% if project_list()|length > 0 -%}} + {{% for project in project_list() -%}} + {project_sql} + {{% if not loop.last %}}UNION ALL{{% endif %}} + {{% endfor %}} + {{%- else %}} + {no_project_sql} + {{%- endif %}} + ) + SELECT + {columns_str}, + FROM + base + """ + return sql def generate_files( filename: str, dir: str, url: str, exclude_columns: List[str], override_table_name: str = None @@ -133,7 +169,7 @@ def generate_files( # Update the column list columns = update_column_list(columns, exclude_columns) - base_filename = f"{dir}/information_schema_{filename}" + base_filename = f"output/{dir}/information_schema_{filename}" # Create the YML file filename_yml = f"{base_filename}.yml" @@ -149,7 +185,7 @@ def generate_files( "version": 2, "models": [ { - "name": table_name.lower(), + "name": "information_schema_" + table_name.lower(), "description": "dataset details with related information", "columns": [ { @@ -179,30 +215,8 @@ def generate_files( # Create the SQL file with open(filename_sql, "w") as f: - f.write( - f"{{# More details about base table in {url} -#}}\n" - + required_role_str - + "WITH base AS (\n" - + "{% if project_list()|length > 0 -%}\n" - + " {% for project in project_list() -%}\n" - + " SELECT * FROM `{{{{ project | trim }}}}`.`region-{{{{ var('bq_region') }}}}`.`INFORMATION_SCHEMA`.`{table_name}`\n".format( - table_name=table_name - ) - + " {% if not loop.last %}UNION ALL{% endif %}\n" - + " {% endfor %}\n" - + "{%- else %}\n" - + " SELECT * FROM `region-{{{{ var('bq_region') }}}}`.`INFORMATION_SCHEMA`.`{table_name}`\n".format( - table_name=table_name - ) - + "{%- endif %}\n" - + ")\n" - + "SELECT\n" - ) - column_names = [column["name"].lower() for column in columns] - f.write(",\n".join(column_names) + ",\n") - - f.write("FROM\n" + " base\n") + f.write(generate_sql(url, column_names, table_name, required_role_str)) print(f"Files '{filename_sql}' and '{filename_yml}' have been created.") diff --git a/documentation_parser/test_documentation_parser.py b/documentation_parser/test_documentation_parser.py index c68216f..63a298e 100644 --- a/documentation_parser/test_documentation_parser.py +++ b/documentation_parser/test_documentation_parser.py @@ -1,6 +1,6 @@ import pytest from bs4 import BeautifulSoup -from documentation_parser import parse_required_role, parse_table_name, update_column_list +from documentation_parser import parse_required_role, parse_table_name, update_column_list, generate_sql def test_parse_required_role(): # Test case: Required role is present @@ -97,3 +97,9 @@ def test_update_column_list(): {'name': 'column3', 'type': 'RECORD', 'description': 'column3.subcolumn1 : Subcolumn 1 of Column 3\ncolumn3.subcolumn2 : Subcolumn 2 of Column 3'}, ] assert result == expected_columns + +def test_generate_sql(): + # Test generate_sql function + result = generate_sql("https://cloud.google.com/bigquery/docs/information-schema-jobs", ["field1", "field2", "field3"], "jobs", "jobs.admin") + expected = "\n {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs -#}\n jobs.admin\n WITH base AS (\n {% if project_list()|length > 0 -%}\n {% for project in project_list() -%}\n \n SELECT field1, field2, field3\n FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`jobs`\n \n {% if not loop.last %}UNION ALL{% endif %}\n {% endfor %}\n {%- else %}\n \n SELECT field1, field2, field3\n FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`jobs`\n \n {%- endif %}\n )\n SELECT\n field1, field2, field3,\n FROM\n base\n " + assert result == expected diff --git a/models/base/google/access_control/information_schema_object_privileges.sql b/models/base/google/access_control/information_schema_object_privileges.sql index 908acad..633b829 100644 --- a/models/base/google/access_control/information_schema_object_privileges.sql +++ b/models/base/google/access_control/information_schema_object_privileges.sql @@ -1,25 +1,30 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-object-privileges -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.OBJECT_PRIVILEGES view, you need following + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-object-privileges -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.OBJECT_PRIVILEGES view, you need following Identity and Access Management (IAM) permissions: bigquery.datasets.get for datasets. bigquery.tables.getIamPolicy for tables and views. For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`OBJECT_PRIVILEGES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`OBJECT_PRIVILEGES` -{%- endif %} -) -SELECT -object_catalog, -object_schema, -object_name, -object_type, -grantee, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT object_catalog, object_schema, object_name, object_type, privilege_type, grantee + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`OBJECT_PRIVILEGES` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT object_catalog, object_schema, object_name, object_type, privilege_type, grantee + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`OBJECT_PRIVILEGES` + + {%- endif %} + ) + SELECT + object_catalog, object_schema, object_name, object_type, privilege_type, grantee, + FROM + base + \ No newline at end of file diff --git a/models/base/google/access_control/information_schema_object_privileges.yml b/models/base/google/access_control/information_schema_object_privileges.yml index a58b16a..ab2b01c 100644 --- a/models/base/google/access_control/information_schema_object_privileges.yml +++ b/models/base/google/access_control/information_schema_object_privileges.yml @@ -13,9 +13,12 @@ models: - description: The resource type, such as SCHEMA (dataset), TABLE, VIEW, and EXTERNAL. name: OBJECT_TYPE type: STRING + - description: The role ID, such as roles/bigquery.dataEditor. + name: PRIVILEGE_TYPE + type: STRING - description: The user type and user that the role is granted to. name: GRANTEE type: STRING description: dataset details with related information - name: object_privileges + name: information_schema_object_privileges version: 2 diff --git a/models/base/google/bi_engine/information_schema_bi_capacities.sql b/models/base/google/bi_engine/information_schema_bi_capacities.sql index 0493585..9aec6bc 100644 --- a/models/base/google/bi_engine/information_schema_bi_capacities.sql +++ b/models/base/google/bi_engine/information_schema_bi_capacities.sql @@ -1,19 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-bi-capacities -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITIES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITIES` -{%- endif %} -) -SELECT -project_id, -project_number, -bi_capacity_name, -size, -preferred_tables, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-bi-capacities -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT project_id, project_number, bi_capacity_name, size, preferred_tables + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITIES` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT project_id, project_number, bi_capacity_name, size, preferred_tables + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITIES` + + {%- endif %} + ) + SELECT + project_id, project_number, bi_capacity_name, size, preferred_tables, + FROM + base + \ No newline at end of file diff --git a/models/base/google/bi_engine/information_schema_bi_capacities.yml b/models/base/google/bi_engine/information_schema_bi_capacities.yml index a327147..fc6aa60 100644 --- a/models/base/google/bi_engine/information_schema_bi_capacities.yml +++ b/models/base/google/bi_engine/information_schema_bi_capacities.yml @@ -20,5 +20,5 @@ models: name: preferred_tables type: REPEATED STRING description: dataset details with related information - name: bi_capacities + name: information_schema_bi_capacities version: 2 diff --git a/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql b/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql index 3b0c8e5..dfea240 100644 --- a/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql +++ b/models/base/google/bi_engine/information_schema_bi_capacity_changes.sql @@ -1,21 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-bi-capacity-changes -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITY_CHANGES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITY_CHANGES` -{%- endif %} -) -SELECT -change_timestamp, -project_id, -project_number, -bi_capacity_name, -size, -user_email, -preferred_tables, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-bi-capacity-changes -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT change_timestamp, project_id, project_number, bi_capacity_name, size, user_email, preferred_tables + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITY_CHANGES` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT change_timestamp, project_id, project_number, bi_capacity_name, size, user_email, preferred_tables + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`BI_CAPACITY_CHANGES` + + {%- endif %} + ) + SELECT + change_timestamp, project_id, project_number, bi_capacity_name, size, user_email, preferred_tables, + FROM + base + \ No newline at end of file diff --git a/models/base/google/bi_engine/information_schema_bi_capacity_changes.yml b/models/base/google/bi_engine/information_schema_bi_capacity_changes.yml index 672b1a6..b76729a 100644 --- a/models/base/google/bi_engine/information_schema_bi_capacity_changes.yml +++ b/models/base/google/bi_engine/information_schema_bi_capacity_changes.yml @@ -29,5 +29,5 @@ models: name: preferred_tables type: REPEATED STRING description: dataset details with related information - name: bi_capacity_changes + name: information_schema_bi_capacity_changes version: 2 diff --git a/models/base/google/configuration/information_schema_effective_project_options.sql b/models/base/google/configuration/information_schema_effective_project_options.sql index ac2bb61..50aaff8 100644 --- a/models/base/google/configuration/information_schema_effective_project_options.sql +++ b/models/base/google/configuration/information_schema_effective_project_options.sql @@ -1,25 +1,30 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-effective-project-options -#} -{# Required role/permissions: To get effective project options metadata, you need the bigquery.config.get + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-effective-project-options -#} + {# Required role/permissions: To get effective project options metadata, you need the bigquery.config.get Identity and Access Management (IAM) permission.The following predefined IAM role includes the permissions that you need in order to get effective project options metadata: roles/bigquery.jobUser For more information about granular BigQuery permissions, see roles and permissions. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`EFFECTIVE_PROJECT_OPTIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`EFFECTIVE_PROJECT_OPTIONS` -{%- endif %} -) -SELECT -option_name, -option_description, -option_type, -option_set_level, -option_set_on_id, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT option_name, option_description, option_type, option_set_level, option_set_on_id + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`EFFECTIVE_PROJECT_OPTIONS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT option_name, option_description, option_type, option_set_level, option_set_on_id + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`EFFECTIVE_PROJECT_OPTIONS` + + {%- endif %} + ) + SELECT + option_name, option_description, option_type, option_set_level, option_set_on_id, + FROM + base + \ No newline at end of file diff --git a/models/base/google/configuration/information_schema_effective_project_options.yml b/models/base/google/configuration/information_schema_effective_project_options.yml index f03b428..c7f1acc 100644 --- a/models/base/google/configuration/information_schema_effective_project_options.yml +++ b/models/base/google/configuration/information_schema_effective_project_options.yml @@ -18,5 +18,5 @@ models: name: OPTION_SET_ON_ID type: STRING description: dataset details with related information - name: effective_project_options + name: information_schema_effective_project_options version: 2 diff --git a/models/base/google/configuration/information_schema_organization_options.sql b/models/base/google/configuration/information_schema_organization_options.sql index 5a5eb75..c955877 100644 --- a/models/base/google/configuration/information_schema_organization_options.sql +++ b/models/base/google/configuration/information_schema_organization_options.sql @@ -1,25 +1,31 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-organization-options -#} -{# Required role/permissions: To get organization options metadata, you need the following Identity and Access Management (IAM) permissions: + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-organization-options -#} + {# Required role/permissions: To get organization options metadata, you need the following Identity and Access Management (IAM) permissions: bigquery.config.get The following predefined IAM role includes the permissions that you need in order to get organization options metadata: roles/bigquery.jobUser For more information about granular BigQuery permissions, see roles and permissions. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS` -{%- endif %} -) -SELECT -option_name, -option_description, -option_type, -option_value, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT option_name, option_description, option_type, option_value + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT option_name, option_description, option_type, option_value + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS` + + {%- endif %} + ) + SELECT + option_name, option_description, option_type, option_value, + FROM + base + \ No newline at end of file diff --git a/models/base/google/configuration/information_schema_organization_options.yml b/models/base/google/configuration/information_schema_organization_options.yml index fb7c434..3ab6946 100644 --- a/models/base/google/configuration/information_schema_organization_options.yml +++ b/models/base/google/configuration/information_schema_organization_options.yml @@ -13,5 +13,5 @@ models: name: OPTION_VALUE type: STRING description: dataset details with related information - name: organization_options + name: information_schema_organization_options version: 2 diff --git a/models/base/google/configuration/information_schema_organization_options_changes.sql b/models/base/google/configuration/information_schema_organization_options_changes.sql index 9f0b85f..00a549a 100644 --- a/models/base/google/configuration/information_schema_organization_options_changes.sql +++ b/models/base/google/configuration/information_schema_organization_options_changes.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-organization-options-changes -#} -{# Required role/permissions: + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-organization-options-changes -#} + {# Required role/permissions: To get the permission that you need to get the configuration changes, @@ -14,21 +15,25 @@ For more information about granting roles, see Manage access. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS_CHANGES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS_CHANGES` -{%- endif %} -) -SELECT -update_time, -username, -updated_options, -project_id, -project_number, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT update_time, username, updated_options, project_id, project_number + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS_CHANGES` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT update_time, username, updated_options, project_id, project_number + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ORGANIZATION_OPTIONS_CHANGES` + + {%- endif %} + ) + SELECT + update_time, username, updated_options, project_id, project_number, + FROM + base + \ No newline at end of file diff --git a/models/base/google/configuration/information_schema_organization_options_changes.yml b/models/base/google/configuration/information_schema_organization_options_changes.yml index cb425ac..e3180d5 100644 --- a/models/base/google/configuration/information_schema_organization_options_changes.yml +++ b/models/base/google/configuration/information_schema_organization_options_changes.yml @@ -20,5 +20,5 @@ models: name: project_number type: INTEGER description: dataset details with related information - name: organization_options_changes + name: information_schema_organization_options_changes version: 2 diff --git a/models/base/google/configuration/information_schema_project_options.sql b/models/base/google/configuration/information_schema_project_options.sql index 5d1ff14..f64fa9a 100644 --- a/models/base/google/configuration/information_schema_project_options.sql +++ b/models/base/google/configuration/information_schema_project_options.sql @@ -1,25 +1,31 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-project-options -#} -{# Required role/permissions: To get configuration options metadata, you need the following Identity and Access Management (IAM) permissions: + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-project-options -#} + {# Required role/permissions: To get configuration options metadata, you need the following Identity and Access Management (IAM) permissions: bigquery.config.get The following predefined IAM role includes the permissions that you need in order to get project options metadata: roles/bigquery.jobUser For more information about granular BigQuery permissions, see roles and permissions. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS` -{%- endif %} -) -SELECT -option_name, -option_description, -option_type, -option_value, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT option_name, option_description, option_type, option_value + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT option_name, option_description, option_type, option_value + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS` + + {%- endif %} + ) + SELECT + option_name, option_description, option_type, option_value, + FROM + base + \ No newline at end of file diff --git a/models/base/google/configuration/information_schema_project_options.yml b/models/base/google/configuration/information_schema_project_options.yml index 79e9c82..9c1ac42 100644 --- a/models/base/google/configuration/information_schema_project_options.yml +++ b/models/base/google/configuration/information_schema_project_options.yml @@ -13,5 +13,5 @@ models: name: OPTION_VALUE type: STRING description: dataset details with related information - name: project_options + name: information_schema_project_options version: 2 diff --git a/models/base/google/configuration/information_schema_project_options_changes.sql b/models/base/google/configuration/information_schema_project_options_changes.sql index 8ffb77f..4d94369 100644 --- a/models/base/google/configuration/information_schema_project_options_changes.sql +++ b/models/base/google/configuration/information_schema_project_options_changes.sql @@ -1,24 +1,29 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-project-options-changes -#} -{# Required role/permissions: To get the configuration, you need the bigquery.config.update + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-project-options-changes -#} + {# Required role/permissions: To get the configuration, you need the bigquery.config.update Identity and Access Management (IAM) permission at the project level. The predefined IAM role roles/bigquery.admin includes the permissions that you need to create a configuration.For more information about granular BigQuery permissions, see roles and permissions. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS_CHANGES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS_CHANGES` -{%- endif %} -) -SELECT -update_time, -username, -updated_options, -project_id, -project_number, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT update_time, username, updated_options, project_id, project_number + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS_CHANGES` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT update_time, username, updated_options, project_id, project_number + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PROJECT_OPTIONS_CHANGES` + + {%- endif %} + ) + SELECT + update_time, username, updated_options, project_id, project_number, + FROM + base + \ No newline at end of file diff --git a/models/base/google/configuration/information_schema_project_options_changes.yml b/models/base/google/configuration/information_schema_project_options_changes.yml index 3e8bfe8..c122621 100644 --- a/models/base/google/configuration/information_schema_project_options_changes.yml +++ b/models/base/google/configuration/information_schema_project_options_changes.yml @@ -20,5 +20,5 @@ models: name: project_number type: INTEGER description: dataset details with related information - name: project_options_changes + name: information_schema_project_options_changes version: 2 diff --git a/models/base/google/datasets/information_schema_links.sql b/models/base/google/datasets/information_schema_links.sql index 7cd0e59..b80a65c 100644 --- a/models/base/google/datasets/information_schema_links.sql +++ b/models/base/google/datasets/information_schema_links.sql @@ -1,21 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata-links -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_LINKS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_LINKS` -{%- endif %} -) -SELECT -catalog_name, -schema_name, -linked_schema_catalog_number, -linked_schema_catalog_name, -linked_schema_name, -linked_schema_creation_time, -linked_schema_org_display_name, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata-links -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT catalog_name, schema_name, linked_schema_catalog_number, linked_schema_catalog_name, linked_schema_name, linked_schema_creation_time, linked_schema_org_display_name + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_LINKS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT catalog_name, schema_name, linked_schema_catalog_number, linked_schema_catalog_name, linked_schema_name, linked_schema_creation_time, linked_schema_org_display_name + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_LINKS` + + {%- endif %} + ) + SELECT + catalog_name, schema_name, linked_schema_catalog_number, linked_schema_catalog_name, linked_schema_name, linked_schema_creation_time, linked_schema_org_display_name, + FROM + base + \ No newline at end of file diff --git a/models/base/google/datasets/information_schema_links.yml b/models/base/google/datasets/information_schema_links.yml index 1dbc308..ff85e7b 100644 --- a/models/base/google/datasets/information_schema_links.yml +++ b/models/base/google/datasets/information_schema_links.yml @@ -25,5 +25,5 @@ models: name: LINKED_SCHEMA_ORG_DISPLAY_NAME type: STRING description: dataset details with related information - name: schemata_links + name: information_schema_schemata_links version: 2 diff --git a/models/base/google/datasets/information_schema_schemata.sql b/models/base/google/datasets/information_schema_schemata.sql index 9af5532..d40cc0a 100644 --- a/models/base/google/datasets/information_schema_schemata.sql +++ b/models/base/google/datasets/information_schema_schemata.sql @@ -1,22 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` -{%- endif %} -) -SELECT -catalog_name, -schema_name, -schema_owner, -creation_time, -last_modified_time, -location, -ddl, -default_collation_name, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT catalog_name, schema_name, schema_owner, creation_time, last_modified_time, location, ddl, default_collation_name + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT catalog_name, schema_name, schema_owner, creation_time, last_modified_time, location, ddl, default_collation_name + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA` + + {%- endif %} + ) + SELECT + catalog_name, schema_name, schema_owner, creation_time, last_modified_time, location, ddl, default_collation_name, + FROM + base + \ No newline at end of file diff --git a/models/base/google/datasets/information_schema_schemata.yml b/models/base/google/datasets/information_schema_schemata.yml index 34b80db..fb55784 100644 --- a/models/base/google/datasets/information_schema_schemata.yml +++ b/models/base/google/datasets/information_schema_schemata.yml @@ -27,5 +27,5 @@ models: name: DEFAULT_COLLATION_NAME type: STRING description: dataset details with related information - name: schemata + name: information_schema_schemata version: 2 diff --git a/models/base/google/datasets/information_schema_schemata_options.sql b/models/base/google/datasets/information_schema_schemata_options.sql index 42ca602..7d8716c 100644 --- a/models/base/google/datasets/information_schema_schemata_options.sql +++ b/models/base/google/datasets/information_schema_schemata_options.sql @@ -1,19 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata-options -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_OPTIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_OPTIONS` -{%- endif %} -) -SELECT -catalog_name, -schema_name, -option_name, -option_type, -option_value, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-datasets-schemata-options -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT catalog_name, schema_name, option_name, option_type, option_value + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_OPTIONS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT catalog_name, schema_name, option_name, option_type, option_value + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_OPTIONS` + + {%- endif %} + ) + SELECT + catalog_name, schema_name, option_name, option_type, option_value, + FROM + base + \ No newline at end of file diff --git a/models/base/google/datasets/information_schema_schemata_options.yml b/models/base/google/datasets/information_schema_schemata_options.yml index 3b2b432..f69feb0 100644 --- a/models/base/google/datasets/information_schema_schemata_options.yml +++ b/models/base/google/datasets/information_schema_schemata_options.yml @@ -7,7 +7,10 @@ models: name: SCHEMA_NAME type: STRING - description: "The name of the option. For a list of supported options, see the\n\ - \ schema options list." + \ schema options list.\n The storage_billing_model option is only\ + \ displayed for\n datasets that have been updated after December 1, 2022.\ + \ For datasets that\n were last updated before that date, the storage billing\ + \ model is\n LOGICAL." name: OPTION_NAME type: STRING - description: The data type of the option @@ -17,5 +20,5 @@ models: name: OPTION_VALUE type: STRING description: dataset details with related information - name: schemata_options + name: information_schema_schemata_options version: 2 diff --git a/models/base/google/datasets/information_schema_schemata_replicas.sql b/models/base/google/datasets/information_schema_schemata_replicas.sql index 0b9e8d4..957a488 100644 --- a/models/base/google/datasets/information_schema_schemata_replicas.sql +++ b/models/base/google/datasets/information_schema_schemata_replicas.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-schemata-replicas -#} -{# Required role/permissions: + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-schemata-replicas -#} + {# Required role/permissions: To get the permissions that you need to query the INFORMATION_SCHEMA.SCHEMATA_REPLICAS view, @@ -19,25 +20,25 @@ roles or other predefined roles. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_REPLICAS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_REPLICAS` -{%- endif %} -) -SELECT -catalog_name, -schema_name, -replica_name, -location, -replica_primary_assigned, -replica_primary_assignment_complete, -creation_time, -creation_complete, -replication_time, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT catalog_name, schema_name, replica_name, location, replica_primary_assigned, replica_primary_assignment_complete, creation_time, creation_complete, replication_time + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_REPLICAS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT catalog_name, schema_name, replica_name, location, replica_primary_assigned, replica_primary_assignment_complete, creation_time, creation_complete, replication_time + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SCHEMATA_REPLICAS` + + {%- endif %} + ) + SELECT + catalog_name, schema_name, replica_name, location, replica_primary_assigned, replica_primary_assignment_complete, creation_time, creation_complete, replication_time, + FROM + base + \ No newline at end of file diff --git a/models/base/google/datasets/information_schema_schemata_replicas.yml b/models/base/google/datasets/information_schema_schemata_replicas.yml index c750eb5..c83c7a3 100644 --- a/models/base/google/datasets/information_schema_schemata_replicas.yml +++ b/models/base/google/datasets/information_schema_schemata_replicas.yml @@ -37,5 +37,5 @@ models: name: replication_time type: TIMESTAMP description: dataset details with related information - name: schemata_replicas + name: information_schema_schemata_replicas version: 2 diff --git a/models/base/google/datasets/information_schema_shared_dataset_usage.sql b/models/base/google/datasets/information_schema_shared_dataset_usage.sql index 904d79b..7daa51c 100644 --- a/models/base/google/datasets/information_schema_shared_dataset_usage.sql +++ b/models/base/google/datasets/information_schema_shared_dataset_usage.sql @@ -1,30 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-shared-dataset-usage -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SHARED_DATASET_USAGE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SHARED_DATASET_USAGE` -{%- endif %} -) -SELECT -project_id, -dataset_id, -table_id, -data_exchange_id, -listing_id, -job_start_time, -job_end_time, -job_id, -job_project_number, -job_location, -linked_project_number, -linked_dataset_id, -subscriber_org_number, -subscriber_org_display_name, -num_rows_processed, -total_bytes_processed, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-shared-dataset-usage -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT project_id, dataset_id, table_id, data_exchange_id, listing_id, job_start_time, job_end_time, job_id, job_project_number, job_location, linked_project_number, linked_dataset_id, subscriber_org_number, subscriber_org_display_name, num_rows_processed, total_bytes_processed + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SHARED_DATASET_USAGE` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT project_id, dataset_id, table_id, data_exchange_id, listing_id, job_start_time, job_end_time, job_id, job_project_number, job_location, linked_project_number, linked_dataset_id, subscriber_org_number, subscriber_org_display_name, num_rows_processed, total_bytes_processed + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SHARED_DATASET_USAGE` + + {%- endif %} + ) + SELECT + project_id, dataset_id, table_id, data_exchange_id, listing_id, job_start_time, job_end_time, job_id, job_project_number, job_location, linked_project_number, linked_dataset_id, subscriber_org_number, subscriber_org_display_name, num_rows_processed, total_bytes_processed, + FROM + base + \ No newline at end of file diff --git a/models/base/google/datasets/information_schema_shared_dataset_usage.yml b/models/base/google/datasets/information_schema_shared_dataset_usage.yml index 99a7f84..e53ae3e 100644 --- a/models/base/google/datasets/information_schema_shared_dataset_usage.yml +++ b/models/base/google/datasets/information_schema_shared_dataset_usage.yml @@ -54,5 +54,5 @@ models: name: total_bytes_processed type: INTEGER description: dataset details with related information - name: shared_dataset_usage + name: information_schema_shared_dataset_usage version: 2 diff --git a/models/base/google/jobs/information_schema_jobs.sql b/models/base/google/jobs/information_schema_jobs.sql index 526e644..210d88f 100644 --- a/models/base/google/jobs/information_schema_jobs.sql +++ b/models/base/google/jobs/information_schema_jobs.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs -#} -{# Required role/permissions: + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs -#} + {# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS view, @@ -14,48 +15,25 @@ For more information about granting roles, see Manage access. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS` -{%- endif %} -) -SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -dml_statistics, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -query_info, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS` + + {%- endif %} + ) + SELECT + bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info, + FROM + base + \ No newline at end of file diff --git a/models/base/google/jobs/information_schema_jobs.yml b/models/base/google/jobs/information_schema_jobs.yml index e6f5312..0d0397d 100644 --- a/models/base/google/jobs/information_schema_jobs.yml +++ b/models/base/google/jobs/information_schema_jobs.yml @@ -150,5 +150,5 @@ models: name: query_info type: RECORD description: dataset details with related information - name: jobs + name: information_schema_jobs version: 2 diff --git a/models/base/google/jobs/information_schema_jobs_by_folder.sql b/models/base/google/jobs/information_schema_jobs_by_folder.sql index feabbb0..d41a807 100644 --- a/models/base/google/jobs/information_schema_jobs_by_folder.sql +++ b/models/base/google/jobs/information_schema_jobs_by_folder.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-folder -#} -{# Required role/permissions: + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-folder -#} + {# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS_BY_FOLDER view, @@ -14,48 +15,25 @@ For more information about granting roles, see Manage access. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_FOLDER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_FOLDER` -{%- endif %} -) -SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -end_time, -error_result, -folder_numbers, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -query_info, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_FOLDER` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_FOLDER` + + {%- endif %} + ) + SELECT + bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info, + FROM + base + \ No newline at end of file diff --git a/models/base/google/jobs/information_schema_jobs_by_folder.yml b/models/base/google/jobs/information_schema_jobs_by_folder.yml index 3244d11..31dd7b0 100644 --- a/models/base/google/jobs/information_schema_jobs_by_folder.yml +++ b/models/base/google/jobs/information_schema_jobs_by_folder.yml @@ -23,14 +23,6 @@ models: - description: Details of any errors as ErrorProto objects. name: error_result type: RECORD - - description: "Number IDs of folders that contain the project, starting with the\ - \ folder that immediately\n contains the project, followed by the folder\ - \ that contains the child folder, and so forth.\n For example, if folder_numbers\ - \ is [1, 2, 3], then folder\n 1 immediately contains the project, folder\ - \ 2 contains\n 1, and folder 3 contains 2. This column is only\n \ - \ populated in JOBS_BY_FOLDER." - name: folder_numbers - type: REPEATED INTEGER - description: The ID of the job. For example, bquxjob_1234. name: job_id type: STRING @@ -150,5 +142,5 @@ models: name: query_info type: RECORD description: dataset details with related information - name: jobs_by_folder + name: information_schema_jobs_by_folder version: 2 diff --git a/models/base/google/jobs/information_schema_jobs_by_organization.sql b/models/base/google/jobs/information_schema_jobs_by_organization.sql index bf146c8..fa189cb 100644 --- a/models/base/google/jobs/information_schema_jobs_by_organization.sql +++ b/models/base/google/jobs/information_schema_jobs_by_organization.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-organization -#} -{# Required role/permissions: + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-organization -#} + {# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS_BY_ORGANIZATION view, @@ -14,47 +15,25 @@ For more information about granting roles, see Manage access. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_ORGANIZATION` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_ORGANIZATION` -{%- endif %} -) -SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -query_info, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_ORGANIZATION` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_ORGANIZATION` + + {%- endif %} + ) + SELECT + bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info, + FROM + base + \ No newline at end of file diff --git a/models/base/google/jobs/information_schema_jobs_by_organization.yml b/models/base/google/jobs/information_schema_jobs_by_organization.yml index 0acae79..5dc2183 100644 --- a/models/base/google/jobs/information_schema_jobs_by_organization.yml +++ b/models/base/google/jobs/information_schema_jobs_by_organization.yml @@ -142,5 +142,5 @@ models: name: query_info type: RECORD description: dataset details with related information - name: jobs_by_organization + name: information_schema_jobs_by_organization version: 2 diff --git a/models/base/google/jobs/information_schema_jobs_by_project.sql b/models/base/google/jobs/information_schema_jobs_by_project.sql index 8a68256..9b2fb2d 100644 --- a/models/base/google/jobs/information_schema_jobs_by_project.sql +++ b/models/base/google/jobs/information_schema_jobs_by_project.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs -#} -{# Required role/permissions: + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs -#} + {# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS view, @@ -14,48 +15,25 @@ For more information about granting roles, see Manage access. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_PROJECT` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_PROJECT` -{%- endif %} -) -SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -dml_statistics, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -query_info, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_PROJECT` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_PROJECT` + + {%- endif %} + ) + SELECT + bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info, + FROM + base + \ No newline at end of file diff --git a/models/base/google/jobs/information_schema_jobs_by_project.yml b/models/base/google/jobs/information_schema_jobs_by_project.yml index b0e0537..dc9da17 100644 --- a/models/base/google/jobs/information_schema_jobs_by_project.yml +++ b/models/base/google/jobs/information_schema_jobs_by_project.yml @@ -150,5 +150,5 @@ models: name: query_info type: RECORD description: dataset details with related information - name: jobs_by_project + name: information_schema_jobs_by_project version: 2 diff --git a/models/base/google/jobs/information_schema_jobs_by_user.sql b/models/base/google/jobs/information_schema_jobs_by_user.sql index f649211..1165289 100644 --- a/models/base/google/jobs/information_schema_jobs_by_user.sql +++ b/models/base/google/jobs/information_schema_jobs_by_user.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-user -#} -{# Required role/permissions: + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-by-user -#} + {# Required role/permissions: To get the permission that you need to query the INFORMATION_SCHEMA.JOBS_BY_USER view, @@ -14,48 +15,25 @@ For more information about granting roles, see Manage access. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_USER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_USER` -{%- endif %} -) -SELECT -bi_engine_statistics, -cache_hit, -creation_time, -destination_table, -dml_statistics, -end_time, -error_result, -job_id, -job_stages, -job_type, -labels, -parent_job_id, -priority, -project_id, -project_number, -query, -referenced_tables, -reservation_id, -session_info, -start_time, -state, -statement_type, -timeline, -total_bytes_billed, -total_bytes_processed, -total_modified_partitions, -total_slot_ms, -transaction_id, -user_email, -transferred_bytes, -materialized_view_statistics, -query_info, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_USER` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_USER` + + {%- endif %} + ) + SELECT + bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info, + FROM + base + \ No newline at end of file diff --git a/models/base/google/jobs/information_schema_jobs_by_user.yml b/models/base/google/jobs/information_schema_jobs_by_user.yml index 93cc2ff..92829dd 100644 --- a/models/base/google/jobs/information_schema_jobs_by_user.yml +++ b/models/base/google/jobs/information_schema_jobs_by_user.yml @@ -150,5 +150,5 @@ models: name: query_info type: RECORD description: dataset details with related information - name: jobs_by_user + name: information_schema_jobs_by_user version: 2 diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql index a3c7232..80a4a8b 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE view, you need the + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the project. Each of the following predefined IAM roles includes the required permission: @@ -7,34 +8,25 @@ Project Owner BigQuery Admin For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE` -{%- endif %} -) -SELECT -period_start, -period_slot_ms, -period_shuffle_ram_usage_ratio, -project_id, -project_number, -user_email, -job_id, -job_type, -statement_type, -job_creation_time, -job_start_time, -job_end_time, -state, -reservation_id, -total_bytes_processed, -error_result, -cache_hit, -period_estimated_runnable_units, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE` + + {%- endif %} + ) + SELECT + period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units, + FROM + base + \ No newline at end of file diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline.yml b/models/base/google/jobs_timeline/information_schema_jobs_timeline.yml index 4ee11c1..89e72a7 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline.yml +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline.yml @@ -64,5 +64,5 @@ models: name: period_estimated_runnable_units type: INTEGER description: dataset details with related information - name: jobs_timeline + name: information_schema_jobs_timeline version: 2 diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql index b6be2d4..0715b08 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.sql @@ -1,42 +1,32 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-folder -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_FOLDER view, you need + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-folder -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_FOLDER view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the parent -folder. -Each of the following predefined IAM roles includes the required -permission: +folder. Each of the following predefined IAM roles includes the +required permission: Folder Admin BigQuery Admin For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_FOLDER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_FOLDER` -{%- endif %} -) -SELECT -period_start, -period_slot_ms, -period_shuffle_ram_usage_ratio, -project_id, -project_number, -folder_numbers, -user_email, -job_id, -job_type, -statement_type, -job_creation_time, -job_start_time, -job_end_time, -state, -reservation_id, -total_bytes_processed, -error_result, -cache_hit, -period_estimated_runnable_units, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, folder_numbers, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_FOLDER` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, folder_numbers, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_FOLDER` + + {%- endif %} + ) + SELECT + period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, folder_numbers, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units, + FROM + base + \ No newline at end of file diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.yml b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.yml index 0ebe5ee..222ddb0 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.yml +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_folder.yml @@ -71,5 +71,5 @@ models: name: period_estimated_runnable_units type: INTEGER description: dataset details with related information - name: jobs_timeline_by_folder + name: information_schema_jobs_timeline_by_folder version: 2 diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql index 3ceab71..7c02da6 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-organization -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_ORGANIZATION view, you need + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-organization -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_ORGANIZATION view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the organization. Each of the following predefined IAM roles includes the required permission: @@ -9,35 +10,25 @@ Organization Admin The JOBS_BY_ORGANIZATION schema table is only available to users with defined Google Cloud organizations.For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_ORGANIZATION` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_ORGANIZATION` -{%- endif %} -) -SELECT -period_start, -period_slot_ms, -period_shuffle_ram_usage_ratio, -project_id, -project_number, -folder_numbers, -user_email, -job_id, -job_type, -statement_type, -job_creation_time, -job_start_time, -job_end_time, -state, -reservation_id, -total_bytes_processed, -error_result, -cache_hit, -period_estimated_runnable_units, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, folder_numbers, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_ORGANIZATION` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, folder_numbers, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_ORGANIZATION` + + {%- endif %} + ) + SELECT + period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, folder_numbers, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units, + FROM + base + \ No newline at end of file diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.yml b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.yml index bd3ade7..da8ffa9 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.yml +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_organization.yml @@ -71,5 +71,5 @@ models: name: period_estimated_runnable_units type: INTEGER description: dataset details with related information - name: jobs_timeline_by_organization + name: information_schema_jobs_timeline_by_organization version: 2 diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql index 4438596..7bd5dc1 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-user -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_USER view, you need the + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-user -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.JOBS_TIMELINE_BY_USER view, you need the bigquery.jobs.list Identity and Access Management (IAM) permission for the project. Each of the following predefined IAM roles includes the required permission: @@ -7,34 +8,25 @@ Project Viewer BigQuery User For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_USER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_USER` -{%- endif %} -) -SELECT -period_start, -period_slot_ms, -period_shuffle_ram_usage_ratio, -project_id, -project_number, -user_email, -job_id, -job_type, -statement_type, -job_creation_time, -job_start_time, -job_end_time, -state, -reservation_id, -total_bytes_processed, -error_result, -cache_hit, -period_estimated_runnable_units, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_USER` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_TIMELINE_BY_USER` + + {%- endif %} + ) + SELECT + period_start, period_slot_ms, period_shuffle_ram_usage_ratio, project_id, project_number, user_email, job_id, job_type, statement_type, job_creation_time, job_start_time, job_end_time, state, reservation_id, total_bytes_processed, error_result, cache_hit, period_estimated_runnable_units, + FROM + base + \ No newline at end of file diff --git a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.yml b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.yml index fb3bdde..52894c9 100644 --- a/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.yml +++ b/models/base/google/jobs_timeline/information_schema_jobs_timeline_by_user.yml @@ -64,5 +64,5 @@ models: name: period_estimated_runnable_units type: INTEGER description: dataset details with related information - name: jobs_timeline_by_user + name: information_schema_jobs_timeline_by_user version: 2 diff --git a/models/base/google/reservations/information_schema_assignment_changes.sql b/models/base/google/reservations/information_schema_assignment_changes.sql index d5963cb..5f5b51f 100644 --- a/models/base/google/reservations/information_schema_assignment_changes.sql +++ b/models/base/google/reservations/information_schema_assignment_changes.sql @@ -1,26 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-assignments-changes -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENT_CHANGES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENT_CHANGES` -{%- endif %} -) -SELECT -change_timestamp, -project_id, -project_number, -assignment_id, -reservation_name, -job_type, -assignee_id, -assignee_number, -assignee_type, -action, -user_email, -state, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-assignments-changes -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT change_timestamp, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type, action, user_email, state + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENT_CHANGES` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT change_timestamp, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type, action, user_email, state + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENT_CHANGES` + + {%- endif %} + ) + SELECT + change_timestamp, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type, action, user_email, state, + FROM + base + \ No newline at end of file diff --git a/models/base/google/reservations/information_schema_assignment_changes.yml b/models/base/google/reservations/information_schema_assignment_changes.yml index 931f539..1254727 100644 --- a/models/base/google/reservations/information_schema_assignment_changes.yml +++ b/models/base/google/reservations/information_schema_assignment_changes.yml @@ -42,5 +42,5 @@ models: name: state type: STRING description: dataset details with related information - name: assignment_changes + name: information_schema_assignment_changes version: 2 diff --git a/models/base/google/reservations/information_schema_assignments.sql b/models/base/google/reservations/information_schema_assignments.sql index afd73d0..29f990f 100644 --- a/models/base/google/reservations/information_schema_assignments.sql +++ b/models/base/google/reservations/information_schema_assignments.sql @@ -1,23 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-assignments -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENTS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENTS` -{%- endif %} -) -SELECT -ddl, -project_id, -project_number, -assignment_id, -reservation_name, -job_type, -assignee_id, -assignee_number, -assignee_type, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-assignments -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT ddl, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENTS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT ddl, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ASSIGNMENTS` + + {%- endif %} + ) + SELECT + ddl, project_id, project_number, assignment_id, reservation_name, job_type, assignee_id, assignee_number, assignee_type, + FROM + base + \ No newline at end of file diff --git a/models/base/google/reservations/information_schema_assignments.yml b/models/base/google/reservations/information_schema_assignments.yml index b52d647..10b4262 100644 --- a/models/base/google/reservations/information_schema_assignments.yml +++ b/models/base/google/reservations/information_schema_assignments.yml @@ -16,7 +16,7 @@ models: name: reservation_name type: STRING - description: "The type of job that can use the reservation. Can be\n PIPELINE,\ - \ QUERY, ML_EXTERNAL, or\n BACKGROUND." + \ QUERY, CONTINUOUS,\n ML_EXTERNAL, or BACKGROUND." name: job_type type: STRING - description: ID that uniquely identifies the assignee resource. @@ -30,5 +30,5 @@ models: name: assignee_type type: STRING description: dataset details with related information - name: assignments + name: information_schema_assignments version: 2 diff --git a/models/base/google/reservations/information_schema_capacity_commitment_changes.sql b/models/base/google/reservations/information_schema_capacity_commitment_changes.sql index dc04edc..93f16ed 100644 --- a/models/base/google/reservations/information_schema_capacity_commitment_changes.sql +++ b/models/base/google/reservations/information_schema_capacity_commitment_changes.sql @@ -1,29 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-capacity-commitment-changes -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENT_CHANGES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENT_CHANGES` -{%- endif %} -) -SELECT -change_timestamp, -project_id, -project_number, -capacity_commitment_id, -commitment_plan, -state, -slot_count, -action, -user_email, -commitment_start_time, -commitment_end_time, -failure_status, -renewal_plan, -edition, -is_flat_rate, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-capacity-commitment-changes -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT change_timestamp, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, action, user_email, commitment_start_time, commitment_end_time, failure_status, renewal_plan, edition, is_flat_rate + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENT_CHANGES` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT change_timestamp, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, action, user_email, commitment_start_time, commitment_end_time, failure_status, renewal_plan, edition, is_flat_rate + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENT_CHANGES` + + {%- endif %} + ) + SELECT + change_timestamp, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, action, user_email, commitment_start_time, commitment_end_time, failure_status, renewal_plan, edition, is_flat_rate, + FROM + base + \ No newline at end of file diff --git a/models/base/google/reservations/information_schema_capacity_commitment_changes.yml b/models/base/google/reservations/information_schema_capacity_commitment_changes.yml index b429622..5c35ff6 100644 --- a/models/base/google/reservations/information_schema_capacity_commitment_changes.yml +++ b/models/base/google/reservations/information_schema_capacity_commitment_changes.yml @@ -59,5 +59,5 @@ models: name: is_flat_rate type: BOOL description: dataset details with related information - name: capacity_commitment_changes + name: information_schema_capacity_commitment_changes version: 2 diff --git a/models/base/google/reservations/information_schema_capacity_commitments.sql b/models/base/google/reservations/information_schema_capacity_commitments.sql index 5beb38d..6511bd7 100644 --- a/models/base/google/reservations/information_schema_capacity_commitments.sql +++ b/models/base/google/reservations/information_schema_capacity_commitments.sql @@ -1,24 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-capacity-commitments -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENTS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENTS` -{%- endif %} -) -SELECT -ddl, -project_id, -project_number, -capacity_commitment_id, -commitment_plan, -state, -slot_count, -edition, -is_flat_rate, -renewal_plan, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-capacity-commitments -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT ddl, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, edition, is_flat_rate, renewal_plan + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENTS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT ddl, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, edition, is_flat_rate, renewal_plan + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CAPACITY_COMMITMENTS` + + {%- endif %} + ) + SELECT + ddl, project_id, project_number, capacity_commitment_id, commitment_plan, state, slot_count, edition, is_flat_rate, renewal_plan, + FROM + base + \ No newline at end of file diff --git a/models/base/google/reservations/information_schema_capacity_commitments.yml b/models/base/google/reservations/information_schema_capacity_commitments.yml index 48f69be..9241f08 100644 --- a/models/base/google/reservations/information_schema_capacity_commitments.yml +++ b/models/base/google/reservations/information_schema_capacity_commitments.yml @@ -36,5 +36,5 @@ models: name: renewal_plan type: STRING description: dataset details with related information - name: capacity_commitments + name: information_schema_capacity_commitments version: 2 diff --git a/models/base/google/reservations/information_schema_reservation_changes.sql b/models/base/google/reservations/information_schema_reservation_changes.sql index bf65758..15d038c 100644 --- a/models/base/google/reservations/information_schema_reservation_changes.sql +++ b/models/base/google/reservations/information_schema_reservation_changes.sql @@ -1,25 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservation-changes -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATION_CHANGES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATION_CHANGES` -{%- endif %} -) -SELECT -change_timestamp, -project_id, -project_number, -reservation_name, -ignore_idle_slots, -action, -slot_capacity, -user_email, -target_job_concurrency, -autoscale, -edition, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservation-changes -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT change_timestamp, project_id, project_number, reservation_name, ignore_idle_slots, action, slot_capacity, user_email, target_job_concurrency, autoscale, edition + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATION_CHANGES` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT change_timestamp, project_id, project_number, reservation_name, ignore_idle_slots, action, slot_capacity, user_email, target_job_concurrency, autoscale, edition + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATION_CHANGES` + + {%- endif %} + ) + SELECT + change_timestamp, project_id, project_number, reservation_name, ignore_idle_slots, action, slot_capacity, user_email, target_job_concurrency, autoscale, edition, + FROM + base + \ No newline at end of file diff --git a/models/base/google/reservations/information_schema_reservation_changes.yml b/models/base/google/reservations/information_schema_reservation_changes.yml index de69a55..d39fb62 100644 --- a/models/base/google/reservations/information_schema_reservation_changes.yml +++ b/models/base/google/reservations/information_schema_reservation_changes.yml @@ -48,5 +48,5 @@ models: name: edition type: STRING description: dataset details with related information - name: reservation_changes + name: information_schema_reservation_changes version: 2 diff --git a/models/base/google/reservations/information_schema_reservations.sql b/models/base/google/reservations/information_schema_reservations.sql index cee38a9..e123782 100644 --- a/models/base/google/reservations/information_schema_reservations.sql +++ b/models/base/google/reservations/information_schema_reservations.sql @@ -1,23 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservations -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS` -{%- endif %} -) -SELECT -ddl, -project_id, -project_number, -reservation_name, -ignore_idle_slots, -slot_capacity, -target_job_concurrency, -autoscale, -edition, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservations -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT ddl, project_id, project_number, reservation_name, ignore_idle_slots, slot_capacity, target_job_concurrency, autoscale, edition + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT ddl, project_id, project_number, reservation_name, ignore_idle_slots, slot_capacity, target_job_concurrency, autoscale, edition + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS` + + {%- endif %} + ) + SELECT + ddl, project_id, project_number, reservation_name, ignore_idle_slots, slot_capacity, target_job_concurrency, autoscale, edition, + FROM + base + \ No newline at end of file diff --git a/models/base/google/reservations/information_schema_reservations.yml b/models/base/google/reservations/information_schema_reservations.yml index f16f76e..1af3203 100644 --- a/models/base/google/reservations/information_schema_reservations.yml +++ b/models/base/google/reservations/information_schema_reservations.yml @@ -39,5 +39,5 @@ models: name: edition type: STRING description: dataset details with related information - name: reservations + name: information_schema_reservations version: 2 diff --git a/models/base/google/reservations/information_schema_reservations_timeline.sql b/models/base/google/reservations/information_schema_reservations_timeline.sql index 12371be..820d380 100644 --- a/models/base/google/reservations/information_schema_reservations_timeline.sql +++ b/models/base/google/reservations/information_schema_reservations_timeline.sql @@ -1,23 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservation-timeline -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS_TIMELINE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS_TIMELINE` -{%- endif %} -) -SELECT -period_start, -project_id, -project_number, -reservation_name, -ignore_idle_slots, -slots_assigned, -slots_max_assigned, -autoscale, -reservation_id, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-reservation-timeline -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT period_start, project_id, project_number, reservation_name, ignore_idle_slots, slots_assigned, slots_max_assigned, autoscale, reservation_id + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS_TIMELINE` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT period_start, project_id, project_number, reservation_name, ignore_idle_slots, slots_assigned, slots_max_assigned, autoscale, reservation_id + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`RESERVATIONS_TIMELINE` + + {%- endif %} + ) + SELECT + period_start, project_id, project_number, reservation_name, ignore_idle_slots, slots_assigned, slots_max_assigned, autoscale, reservation_id, + FROM + base + \ No newline at end of file diff --git a/models/base/google/reservations/information_schema_reservations_timeline.yml b/models/base/google/reservations/information_schema_reservations_timeline.yml index b41ad6f..49fd27e 100644 --- a/models/base/google/reservations/information_schema_reservations_timeline.yml +++ b/models/base/google/reservations/information_schema_reservations_timeline.yml @@ -38,5 +38,5 @@ models: name: reservation_id type: STRING description: dataset details with related information - name: reservations_timeline + name: information_schema_reservations_timeline version: 2 diff --git a/models/base/google/routines/information_schema_parameters.sql b/models/base/google/routines/information_schema_parameters.sql index 112aab4..97bfdff 100644 --- a/models/base/google/routines/information_schema_parameters.sql +++ b/models/base/google/routines/information_schema_parameters.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-parameters -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.PARAMETERS view, you need the following + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-parameters -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.PARAMETERS view, you need the following Identity and Access Management (IAM) permissions: bigquery.routines.get bigquery.routines.list @@ -10,26 +11,25 @@ roles/bigquery.metadataViewer roles/bigquery.dataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PARAMETERS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PARAMETERS` -{%- endif %} -) -SELECT -specific_catalog, -specific_schema, -specific_name, -ordinal_position, -parameter_mode, -is_result, -parameter_name, -data_type, -parameter_default, -is_aggregate, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT specific_catalog, specific_schema, specific_name, ordinal_position, parameter_mode, is_result, parameter_name, data_type, parameter_default, is_aggregate + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PARAMETERS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT specific_catalog, specific_schema, specific_name, ordinal_position, parameter_mode, is_result, parameter_name, data_type, parameter_default, is_aggregate + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`PARAMETERS` + + {%- endif %} + ) + SELECT + specific_catalog, specific_schema, specific_name, ordinal_position, parameter_mode, is_result, parameter_name, data_type, parameter_default, is_aggregate, + FROM + base + \ No newline at end of file diff --git a/models/base/google/routines/information_schema_parameters.yml b/models/base/google/routines/information_schema_parameters.yml index 10314ee..7ed807b 100644 --- a/models/base/google/routines/information_schema_parameters.yml +++ b/models/base/google/routines/information_schema_parameters.yml @@ -36,5 +36,5 @@ models: name: IS_AGGREGATE type: STRING description: dataset details with related information - name: parameters + name: information_schema_parameters version: 2 diff --git a/models/base/google/routines/information_schema_routine_options.sql b/models/base/google/routines/information_schema_routine_options.sql index c3947d6..2919f93 100644 --- a/models/base/google/routines/information_schema_routine_options.sql +++ b/models/base/google/routines/information_schema_routine_options.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-routine-options -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.ROUTINE_OPTIONS view, you need the following + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-routine-options -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.ROUTINE_OPTIONS view, you need the following Identity and Access Management (IAM) permissions: bigquery.routines.get bigquery.routines.list @@ -10,22 +11,25 @@ roles/bigquery.metadataViewer roles/bigquery.dataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINE_OPTIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINE_OPTIONS` -{%- endif %} -) -SELECT -specific_catalog, -specific_schema, -specific_name, -option_name, -option_type, -option_value, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT specific_catalog, specific_schema, specific_name, option_name, option_type, option_value + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINE_OPTIONS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT specific_catalog, specific_schema, specific_name, option_name, option_type, option_value + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINE_OPTIONS` + + {%- endif %} + ) + SELECT + specific_catalog, specific_schema, specific_name, option_name, option_type, option_value, + FROM + base + \ No newline at end of file diff --git a/models/base/google/routines/information_schema_routine_options.yml b/models/base/google/routines/information_schema_routine_options.yml index 31db34a..2f2a652 100644 --- a/models/base/google/routines/information_schema_routine_options.yml +++ b/models/base/google/routines/information_schema_routine_options.yml @@ -21,5 +21,5 @@ models: name: OPTION_VALUE type: STRING description: dataset details with related information - name: routine_options + name: information_schema_routine_options version: 2 diff --git a/models/base/google/routines/information_schema_routines.sql b/models/base/google/routines/information_schema_routines.sql index 0ac977a..a6095fd 100644 --- a/models/base/google/routines/information_schema_routines.sql +++ b/models/base/google/routines/information_schema_routines.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-routines -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.ROUTINES view, you need the following + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-routines -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.ROUTINES view, you need the following Identity and Access Management (IAM) permissions: bigquery.routines.get bigquery.routines.list @@ -10,32 +11,25 @@ roles/bigquery.metadataViewer roles/bigquery.dataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINE` -{%- endif %} -) -SELECT -specific_catalog, -specific_schema, -specific_name, -routine_catalog, -routine_schema, -routine_name, -routine_type, -data_type, -routine_body, -routine_definition, -external_language, -is_deterministic, -security_type, -created, -last_altered, -ddl, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT specific_catalog, specific_schema, specific_name, routine_catalog, routine_schema, routine_name, routine_type, data_type, routine_body, routine_definition, external_language, is_deterministic, security_type, created, last_altered, ddl + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINES` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT specific_catalog, specific_schema, specific_name, routine_catalog, routine_schema, routine_name, routine_type, data_type, routine_body, routine_definition, external_language, is_deterministic, security_type, created, last_altered, ddl + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`ROUTINES` + + {%- endif %} + ) + SELECT + specific_catalog, specific_schema, specific_name, routine_catalog, routine_schema, routine_name, routine_type, data_type, routine_body, routine_definition, external_language, is_deterministic, security_type, created, last_altered, ddl, + FROM + base + \ No newline at end of file diff --git a/models/base/google/routines/information_schema_routines.yml b/models/base/google/routines/information_schema_routines.yml index f1f94a6..ed47e39 100644 --- a/models/base/google/routines/information_schema_routines.yml +++ b/models/base/google/routines/information_schema_routines.yml @@ -58,5 +58,5 @@ models: name: DDL type: STRING description: dataset details with related information - name: routine + name: information_schema_routines version: 2 diff --git a/models/base/google/search_indexes/information_schema_search_index_columns.sql b/models/base/google/search_indexes/information_schema_search_index_columns.sql index 8e8c9c1..aa636a6 100644 --- a/models/base/google/search_indexes/information_schema_search_index_columns.sql +++ b/models/base/google/search_indexes/information_schema_search_index_columns.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-index-columns -#} -{# Required role/permissions: To see search index metadata, you need the + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-index-columns -#} + {# Required role/permissions: To see search index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) permission on the table with the index. Each of the following predefined IAM roles includes at least one of these permissions: @@ -11,22 +12,25 @@ roles/bigquery.metadataViewer roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEX_COLUMNS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEX_COLUMNS` -{%- endif %} -) -SELECT -index_catalog, -index_schema, -table_name, -index_name, -index_column_name, -index_field_path, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT index_catalog, index_schema, table_name, index_name, index_column_name, index_field_path + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEX_COLUMNS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT index_catalog, index_schema, table_name, index_name, index_column_name, index_field_path + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEX_COLUMNS` + + {%- endif %} + ) + SELECT + index_catalog, index_schema, table_name, index_name, index_column_name, index_field_path, + FROM + base + \ No newline at end of file diff --git a/models/base/google/search_indexes/information_schema_search_index_columns.yml b/models/base/google/search_indexes/information_schema_search_index_columns.yml index dddc231..89b4d8d 100644 --- a/models/base/google/search_indexes/information_schema_search_index_columns.yml +++ b/models/base/google/search_indexes/information_schema_search_index_columns.yml @@ -20,5 +20,5 @@ models: name: index_field_path type: STRING description: dataset details with related information - name: search_index_columns + name: information_schema_search_index_columns version: 2 diff --git a/models/base/google/search_indexes/information_schema_search_indexes.sql b/models/base/google/search_indexes/information_schema_search_indexes.sql index f2a457e..1e20479 100644 --- a/models/base/google/search_indexes/information_schema_search_indexes.sql +++ b/models/base/google/search_indexes/information_schema_search_indexes.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-indexes -#} -{# Required role/permissions: To see search index metadata, you need the + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-indexes -#} + {# Required role/permissions: To see search index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) permission on the table with the index. Each of the following predefined IAM roles includes at least one of these permissions: @@ -11,32 +12,25 @@ roles/bigquery.metadataViewer roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEXES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEXES` -{%- endif %} -) -SELECT -index_catalog, -index_schema, -table_name, -index_name, -index_status, -creation_time, -last_modification_time, -last_refresh_time, -disable_time, -disable_reason, -ddl, -coverage_percentage, -unindexed_row_count, -total_logical_bytes, -total_storage_bytes, -analyzer, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT index_catalog, index_schema, table_name, index_name, index_status, creation_time, last_modification_time, last_refresh_time, disable_time, disable_reason, ddl, coverage_percentage, unindexed_row_count, total_logical_bytes, total_storage_bytes, analyzer + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEXES` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT index_catalog, index_schema, table_name, index_name, index_status, creation_time, last_modification_time, last_refresh_time, disable_time, disable_reason, ddl, coverage_percentage, unindexed_row_count, total_logical_bytes, total_storage_bytes, analyzer + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SEARCH_INDEXES` + + {%- endif %} + ) + SELECT + index_catalog, index_schema, table_name, index_name, index_status, creation_time, last_modification_time, last_refresh_time, disable_time, disable_reason, ddl, coverage_percentage, unindexed_row_count, total_logical_bytes, total_storage_bytes, analyzer, + FROM + base + \ No newline at end of file diff --git a/models/base/google/search_indexes/information_schema_search_indexes.yml b/models/base/google/search_indexes/information_schema_search_indexes.yml index 090a0d6..0c3ed1e 100644 --- a/models/base/google/search_indexes/information_schema_search_indexes.yml +++ b/models/base/google/search_indexes/information_schema_search_indexes.yml @@ -71,5 +71,5 @@ models: name: analyzer type: STRING description: dataset details with related information - name: search_indexes + name: information_schema_search_indexes version: 2 diff --git a/models/base/google/sessions/information_schema_sessions_by_project.sql b/models/base/google/sessions/information_schema_sessions_by_project.sql index e1adfd1..7411d48 100644 --- a/models/base/google/sessions/information_schema_sessions_by_project.sql +++ b/models/base/google/sessions/information_schema_sessions_by_project.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-sessions-by-project -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.SESSIONS_BY_PROJECT view, you need + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-sessions-by-project -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.SESSIONS_BY_PROJECT view, you need the bigquery.jobs.listAll Identity and Access Management (IAM) permission for the project. Each of the following predefined IAM roles includes the required permission: @@ -7,25 +8,25 @@ Project Owner BigQuery Admin For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_PROJECT` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_PROJECT` -{%- endif %} -) -SELECT -creation_time, -expiration_time, -is_active, -last_modified_time, -principal_subject, -project_id, -project_number, -session_id, -user_email, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT creation_time, expiration_time, is_active, last_modified_time, principal_subject, project_id, project_number, session_id, user_email + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_PROJECT` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT creation_time, expiration_time, is_active, last_modified_time, principal_subject, project_id, project_number, session_id, user_email + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_PROJECT` + + {%- endif %} + ) + SELECT + creation_time, expiration_time, is_active, last_modified_time, principal_subject, project_id, project_number, session_id, user_email, + FROM + base + \ No newline at end of file diff --git a/models/base/google/sessions/information_schema_sessions_by_project.yml b/models/base/google/sessions/information_schema_sessions_by_project.yml index 2e861a5..0a3ddf7 100644 --- a/models/base/google/sessions/information_schema_sessions_by_project.yml +++ b/models/base/google/sessions/information_schema_sessions_by_project.yml @@ -34,5 +34,5 @@ models: name: user_email type: STRING description: dataset details with related information - name: sessions_by_project + name: information_schema_sessions_by_project version: 2 diff --git a/models/base/google/sessions/information_schema_sessions_by_user.sql b/models/base/google/sessions/information_schema_sessions_by_user.sql index 4125d0c..53af611 100644 --- a/models/base/google/sessions/information_schema_sessions_by_user.sql +++ b/models/base/google/sessions/information_schema_sessions_by_user.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-sessions-by-user -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.SESSIONS_BY_USER view, you need + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-sessions-by-user -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.SESSIONS_BY_USER view, you need the bigquery.jobs.list Identity and Access Management (IAM) permission for the project. Each of the following predefined IAM roles includes the required permission: @@ -7,24 +8,25 @@ Project Viewer BigQuery User For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_USER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_USER` -{%- endif %} -) -SELECT -creation_time, -expiration_time, -is_active, -last_modified_time, -project_id, -project_number, -session_id, -user_email, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT creation_time, expiration_time, is_active, last_modified_time, project_id, project_number, session_id, user_email + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_USER` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT creation_time, expiration_time, is_active, last_modified_time, project_id, project_number, session_id, user_email + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`SESSIONS_BY_USER` + + {%- endif %} + ) + SELECT + creation_time, expiration_time, is_active, last_modified_time, project_id, project_number, session_id, user_email, + FROM + base + \ No newline at end of file diff --git a/models/base/google/sessions/information_schema_sessions_by_user.yml b/models/base/google/sessions/information_schema_sessions_by_user.yml index 6f82879..13a5eb8 100644 --- a/models/base/google/sessions/information_schema_sessions_by_user.yml +++ b/models/base/google/sessions/information_schema_sessions_by_user.yml @@ -30,5 +30,5 @@ models: name: user_email type: STRING description: dataset details with related information - name: sessions_by_user + name: information_schema_sessions_by_user version: 2 diff --git a/models/base/google/streaming/information_schema_streaming_timeline.sql b/models/base/google/streaming/information_schema_streaming_timeline.sql index 22c9c67..585ffbd 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline.sql +++ b/models/base/google/streaming/information_schema_streaming_timeline.sql @@ -1,23 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE` -{%- endif %} -) -SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -error_code, -total_requests, -total_rows, -total_input_bytes, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE` + + {%- endif %} + ) + SELECT + start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes, + FROM + base + \ No newline at end of file diff --git a/models/base/google/streaming/information_schema_streaming_timeline.yml b/models/base/google/streaming/information_schema_streaming_timeline.yml index bf7c87f..853997f 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline.yml +++ b/models/base/google/streaming/information_schema_streaming_timeline.yml @@ -30,5 +30,5 @@ models: name: total_input_bytes type: INTEGER description: dataset details with related information - name: streaming_timeline + name: information_schema_streaming_timeline version: 2 diff --git a/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql b/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql index 4614b71..ba27156 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql +++ b/models/base/google/streaming/information_schema_streaming_timeline_by_folder.sql @@ -1,23 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming-by-folder -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_FOLDER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_FOLDER` -{%- endif %} -) -SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -error_code, -total_requests, -total_rows, -total_input_bytes, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming-by-folder -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_FOLDER` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_FOLDER` + + {%- endif %} + ) + SELECT + start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes, + FROM + base + \ No newline at end of file diff --git a/models/base/google/streaming/information_schema_streaming_timeline_by_folder.yml b/models/base/google/streaming/information_schema_streaming_timeline_by_folder.yml index 8dbaf3e..20cc633 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline_by_folder.yml +++ b/models/base/google/streaming/information_schema_streaming_timeline_by_folder.yml @@ -30,5 +30,5 @@ models: name: total_input_bytes type: INTEGER description: dataset details with related information - name: streaming_timeline_by_folder + name: information_schema_streaming_timeline_by_folder version: 2 diff --git a/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql b/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql index 07b2628..ce45288 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql +++ b/models/base/google/streaming/information_schema_streaming_timeline_by_organization.sql @@ -1,23 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming-by-organization -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_ORGANIZATION` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_ORGANIZATION` -{%- endif %} -) -SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -error_code, -total_requests, -total_rows, -total_input_bytes, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-streaming-by-organization -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_ORGANIZATION` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`STREAMING_TIMELINE_BY_ORGANIZATION` + + {%- endif %} + ) + SELECT + start_timestamp, project_id, project_number, dataset_id, table_id, error_code, total_requests, total_rows, total_input_bytes, + FROM + base + \ No newline at end of file diff --git a/models/base/google/streaming/information_schema_streaming_timeline_by_organization.yml b/models/base/google/streaming/information_schema_streaming_timeline_by_organization.yml index 9540050..86e6a9e 100644 --- a/models/base/google/streaming/information_schema_streaming_timeline_by_organization.yml +++ b/models/base/google/streaming/information_schema_streaming_timeline_by_organization.yml @@ -30,5 +30,5 @@ models: name: total_input_bytes type: INTEGER description: dataset details with related information - name: streaming_timeline_by_organization + name: information_schema_streaming_timeline_by_organization version: 2 diff --git a/models/base/google/tables/information_schema_column_field_paths.sql b/models/base/google/tables/information_schema_column_field_paths.sql index 83ee7e5..ae0b4ac 100644 --- a/models/base/google/tables/information_schema_column_field_paths.sql +++ b/models/base/google/tables/information_schema_column_field_paths.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-column-field-paths -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.COLUMN_FIELD_PATHS view, you need the following + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-column-field-paths -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.COLUMN_FIELD_PATHS view, you need the following Identity and Access Management (IAM) permissions: bigquery.tables.get bigquery.tables.list @@ -11,25 +12,25 @@ roles/bigquery.dataEditor roles/bigquery.metadataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMN_FIELD_PATHS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMN_FIELD_PATHS` -{%- endif %} -) -SELECT -table_catalog, -table_schema, -table_name, -column_name, -field_path, -data_type, -description, -collation_name, -rounding_mode, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT table_catalog, table_schema, table_name, column_name, field_path, data_type, description, collation_name, rounding_mode + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMN_FIELD_PATHS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT table_catalog, table_schema, table_name, column_name, field_path, data_type, description, collation_name, rounding_mode + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMN_FIELD_PATHS` + + {%- endif %} + ) + SELECT + table_catalog, table_schema, table_name, column_name, field_path, data_type, description, collation_name, rounding_mode, + FROM + base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_column_field_paths.yml b/models/base/google/tables/information_schema_column_field_paths.yml index 9ec5bdf..ca9e9d9 100644 --- a/models/base/google/tables/information_schema_column_field_paths.yml +++ b/models/base/google/tables/information_schema_column_field_paths.yml @@ -35,5 +35,5 @@ models: name: ROUNDING_MODE type: STRING description: dataset details with related information - name: column_field_paths + name: information_schema_column_field_paths version: 2 diff --git a/models/base/google/tables/information_schema_columns.sql b/models/base/google/tables/information_schema_columns.sql index 8d4a6eb..bd80440 100644 --- a/models/base/google/tables/information_schema_columns.sql +++ b/models/base/google/tables/information_schema_columns.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-columns -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.COLUMNS view, you need the following + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-columns -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.COLUMNS view, you need the following Identity and Access Management (IAM) permissions: bigquery.tables.get bigquery.tables.list @@ -11,34 +12,25 @@ roles/bigquery.dataEditor roles/bigquery.metadataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMNS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMNS` -{%- endif %} -) -SELECT -table_catalog, -table_schema, -table_name, -column_name, -ordinal_position, -is_nullable, -data_type, -is_generated, -generation_expression, -is_stored, -is_hidden, -is_updatable, -is_system_defined, -is_partitioning_column, -clustering_ordinal_position, -collation_name, -column_default, -rounding_mode, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT table_catalog, table_schema, table_name, column_name, ordinal_position, is_nullable, data_type, is_generated, generation_expression, is_stored, is_hidden, is_updatable, is_system_defined, is_partitioning_column, clustering_ordinal_position, collation_name, column_default, rounding_mode + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMNS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT table_catalog, table_schema, table_name, column_name, ordinal_position, is_nullable, data_type, is_generated, generation_expression, is_stored, is_hidden, is_updatable, is_system_defined, is_partitioning_column, clustering_ordinal_position, collation_name, column_default, rounding_mode + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`COLUMNS` + + {%- endif %} + ) + SELECT + table_catalog, table_schema, table_name, column_name, ordinal_position, is_nullable, data_type, is_generated, generation_expression, is_stored, is_hidden, is_updatable, is_system_defined, is_partitioning_column, clustering_ordinal_position, collation_name, column_default, rounding_mode, + FROM + base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_columns.yml b/models/base/google/tables/information_schema_columns.yml index 9aebcb8..4a59931 100644 --- a/models/base/google/tables/information_schema_columns.yml +++ b/models/base/google/tables/information_schema_columns.yml @@ -68,5 +68,5 @@ models: name: ROUNDING_MODE type: STRING description: dataset details with related information - name: columns + name: information_schema_columns version: 2 diff --git a/models/base/google/tables/information_schema_constraint_column_usage.sql b/models/base/google/tables/information_schema_constraint_column_usage.sql index bba788e..5893463 100644 --- a/models/base/google/tables/information_schema_constraint_column_usage.sql +++ b/models/base/google/tables/information_schema_constraint_column_usage.sql @@ -1,21 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-constraint-column-usage -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CONSTRAINT_COLUMN_USAGE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CONSTRAINT_COLUMN_USAGE` -{%- endif %} -) -SELECT -table_catalog, -table_schema, -table_name, -column_name, -constraint_catalog, -constraint_schema, -constraint_name, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-constraint-column-usage -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT table_catalog, table_schema, table_name, column_name, constraint_catalog, constraint_schema, constraint_name + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CONSTRAINT_COLUMN_USAGE` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT table_catalog, table_schema, table_name, column_name, constraint_catalog, constraint_schema, constraint_name + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`CONSTRAINT_COLUMN_USAGE` + + {%- endif %} + ) + SELECT + table_catalog, table_schema, table_name, column_name, constraint_catalog, constraint_schema, constraint_name, + FROM + base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_constraint_column_usage.yml b/models/base/google/tables/information_schema_constraint_column_usage.yml index 748f84d..795c065 100644 --- a/models/base/google/tables/information_schema_constraint_column_usage.yml +++ b/models/base/google/tables/information_schema_constraint_column_usage.yml @@ -25,5 +25,5 @@ models: name: CONSTRAINT_NAME type: STRING description: dataset details with related information - name: constraint_column_usage + name: information_schema_constraint_column_usage version: 2 diff --git a/models/base/google/tables/information_schema_key_column_usage.sql b/models/base/google/tables/information_schema_key_column_usage.sql index d2b43b2..6b87353 100644 --- a/models/base/google/tables/information_schema_key_column_usage.sql +++ b/models/base/google/tables/information_schema_key_column_usage.sql @@ -1,23 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-key-column-usage -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`KEY_COLUMN_USAGE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`KEY_COLUMN_USAGE` -{%- endif %} -) -SELECT -constraint_catalog, -constraint_schema, -constraint_name, -table_catalog, -table_schema, -table_name, -column_name, -ordinal_position, -position_in_unique_constraint, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-key-column-usage -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, column_name, ordinal_position, position_in_unique_constraint + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`KEY_COLUMN_USAGE` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, column_name, ordinal_position, position_in_unique_constraint + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`KEY_COLUMN_USAGE` + + {%- endif %} + ) + SELECT + constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, column_name, ordinal_position, position_in_unique_constraint, + FROM + base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_key_column_usage.yml b/models/base/google/tables/information_schema_key_column_usage.yml index b4aa404..d0dc84b 100644 --- a/models/base/google/tables/information_schema_key_column_usage.yml +++ b/models/base/google/tables/information_schema_key_column_usage.yml @@ -34,5 +34,5 @@ models: name: POSITION_IN_UNIQUE_CONSTRAINT type: INT64 description: dataset details with related information - name: key_column_usage + name: information_schema_key_column_usage version: 2 diff --git a/models/base/google/tables/information_schema_table_constraints.sql b/models/base/google/tables/information_schema_table_constraints.sql index 9233327..9e70965 100644 --- a/models/base/google/tables/information_schema_table_constraints.sql +++ b/models/base/google/tables/information_schema_table_constraints.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-constraints -#} -{# Required role/permissions: You need the following + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-constraints -#} + {# Required role/permissions: You need the following Identity and Access Management (IAM) permissions: bigquery.tables.get for viewing primary and foreign key definitions. bigquery.tables.list for viewing table information schemas. @@ -14,26 +15,25 @@ recommend that you use predefined roles from earlier in the list to not allocate excess permissions.For more information about IAM roles and permissions in BigQuery, see Predefined roles and permissions. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_CONSTRAINTS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_CONSTRAINTS` -{%- endif %} -) -SELECT -constraint_catalog, -constraint_schema, -constraint_name, -table_catalog, -table_schema, -table_name, -constraint_type, -is_deferrable, -initially_deferred, -enforced, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, constraint_type, is_deferrable, initially_deferred, enforced + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_CONSTRAINTS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, constraint_type, is_deferrable, initially_deferred, enforced + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_CONSTRAINTS` + + {%- endif %} + ) + SELECT + constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, constraint_type, is_deferrable, initially_deferred, enforced, + FROM + base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_table_constraints.yml b/models/base/google/tables/information_schema_table_constraints.yml index 2b6de6b..db81889 100644 --- a/models/base/google/tables/information_schema_table_constraints.yml +++ b/models/base/google/tables/information_schema_table_constraints.yml @@ -35,5 +35,5 @@ models: name: ENFORCED type: STRING description: dataset details with related information - name: table_constraints + name: information_schema_table_constraints version: 2 diff --git a/models/base/google/tables/information_schema_table_options.sql b/models/base/google/tables/information_schema_table_options.sql index b6d67ed..f6ebd94 100644 --- a/models/base/google/tables/information_schema_table_options.sql +++ b/models/base/google/tables/information_schema_table_options.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-options -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_OPTIONS view, you need the following + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-options -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_OPTIONS view, you need the following Identity and Access Management (IAM) permissions: bigquery.tables.get bigquery.tables.list @@ -12,22 +13,25 @@ roles/bigquery.dataViewer roles/bigquery.metadataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_OPTIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_OPTIONS` -{%- endif %} -) -SELECT -table_catalog, -table_schema, -table_name, -option_name, -option_type, -option_value, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT table_catalog, table_schema, table_name, option_name, option_type, option_value + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_OPTIONS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT table_catalog, table_schema, table_name, option_name, option_type, option_value + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_OPTIONS` + + {%- endif %} + ) + SELECT + table_catalog, table_schema, table_name, option_name, option_type, option_value, + FROM + base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_table_options.yml b/models/base/google/tables/information_schema_table_options.yml index 5a2373b..e705fa6 100644 --- a/models/base/google/tables/information_schema_table_options.yml +++ b/models/base/google/tables/information_schema_table_options.yml @@ -20,5 +20,5 @@ models: name: OPTION_VALUE type: STRING description: dataset details with related information - name: table_options + name: information_schema_table_options version: 2 diff --git a/models/base/google/tables/information_schema_table_snapshots.sql b/models/base/google/tables/information_schema_table_snapshots.sql index 00c3f15..a8cb6f3 100644 --- a/models/base/google/tables/information_schema_table_snapshots.sql +++ b/models/base/google/tables/information_schema_table_snapshots.sql @@ -1,26 +1,29 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-snapshots -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_SNAPSHOTS view, you need the + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-snapshots -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_SNAPSHOTS view, you need the bigquery.tables.list Identity and Access Management (IAM) permission for the dataset. The roles/bigquery.metadataViewer predefined role includes the required permission.For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_SNAPSHOTS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_SNAPSHOTS` -{%- endif %} -) -SELECT -table_catalog, -table_schema, -table_name, -base_table_catalog, -base_table_schema, -base_table_name, -snapshot_time, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT table_catalog, table_schema, table_name, base_table_catalog, base_table_schema, base_table_name, snapshot_time + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_SNAPSHOTS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT table_catalog, table_schema, table_name, base_table_catalog, base_table_schema, base_table_name, snapshot_time + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_SNAPSHOTS` + + {%- endif %} + ) + SELECT + table_catalog, table_schema, table_name, base_table_catalog, base_table_schema, base_table_name, snapshot_time, + FROM + base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_table_snapshots.yml b/models/base/google/tables/information_schema_table_snapshots.yml index dd3ef58..f1e5a91 100644 --- a/models/base/google/tables/information_schema_table_snapshots.yml +++ b/models/base/google/tables/information_schema_table_snapshots.yml @@ -22,5 +22,5 @@ models: name: snapshot_time type: TIMESTAMP description: dataset details with related information - name: table_snapshots + name: information_schema_table_snapshots version: 2 diff --git a/models/base/google/tables/information_schema_table_storage.sql b/models/base/google/tables/information_schema_table_storage.sql index 083ab29..6f9e749 100644 --- a/models/base/google/tables/information_schema_table_storage.sql +++ b/models/base/google/tables/information_schema_table_storage.sql @@ -1,33 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE` -{%- endif %} -) -SELECT -project_id, -table_catalog, -project_number, -table_schema, -table_name, -creation_time, -deleted, -storage_last_modified_time, -total_rows, -total_partitions, -total_logical_bytes, -active_logical_bytes, -long_term_logical_bytes, -total_physical_bytes, -active_physical_bytes, -long_term_physical_bytes, -time_travel_physical_bytes, -fail_safe_physical_bytes, -table_type, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT project_id, project_number, table_catalog, table_schema, table_name, creation_time, total_rows, total_partitions, total_logical_bytes, active_logical_bytes, long_term_logical_bytes, current_physical_bytes, total_physical_bytes, active_physical_bytes, long_term_physical_bytes, time_travel_physical_bytes, storage_last_modified_time, deleted, table_type, fail_safe_physical_bytes + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT project_id, project_number, table_catalog, table_schema, table_name, creation_time, total_rows, total_partitions, total_logical_bytes, active_logical_bytes, long_term_logical_bytes, current_physical_bytes, total_physical_bytes, active_physical_bytes, long_term_physical_bytes, time_travel_physical_bytes, storage_last_modified_time, deleted, table_type, fail_safe_physical_bytes + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE` + + {%- endif %} + ) + SELECT + project_id, project_number, table_catalog, table_schema, table_name, creation_time, total_rows, total_partitions, total_logical_bytes, active_logical_bytes, long_term_logical_bytes, current_physical_bytes, total_physical_bytes, active_physical_bytes, long_term_physical_bytes, time_travel_physical_bytes, storage_last_modified_time, deleted, table_type, fail_safe_physical_bytes, + FROM + base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_table_storage.yml b/models/base/google/tables/information_schema_table_storage.yml index c3deb65..d532be7 100644 --- a/models/base/google/tables/information_schema_table_storage.yml +++ b/models/base/google/tables/information_schema_table_storage.yml @@ -1,73 +1,76 @@ models: - columns: - - description: The project ID of the project that contains the dataset + - description: The project ID of the project that contains the dataset. name: PROJECT_ID type: STRING - - description: The project ID of the project that contains the dataset - name: TABLE_CATALOG - type: STRING - - description: The project number of the project that contains the dataset + - description: The project number of the project that contains the dataset. name: PROJECT_NUMBER type: INT64 + - description: The project ID of the project that contains the dataset. + name: TABLE_CATALOG + type: STRING - description: "The name of the dataset that contains the table or materialized\ - \ view,\n also referred to as the datasetId" + \ view,\n also referred to as the datasetId." name: TABLE_SCHEMA type: STRING - description: "The name of the table or materialized view, also referred to as\ - \ the\n tableId" + \ the\n tableId." name: TABLE_NAME type: STRING - - description: The table's creation time + - description: The creation time of the table. name: CREATION_TIME type: TIMESTAMP - - description: Indicates whether or not the table is deleted - name: DELETED - type: BOOLEAN - - description: The most recent time that data was written to the table. - name: STORAGE_LAST_MODIFIED_TIME - type: TIMESTAMP - - description: The total number of rows in the table or materialized view + - description: The total number of rows in the table or materialized view. name: TOTAL_ROWS type: INT64 - description: "The number of partitions present in the table or materialized view.\n\ \ Unpartitioned tables return 0." name: TOTAL_PARTITIONS type: INT64 - - description: Total number of logical (uncompressed) bytes in the table or materialized - view + - description: "Total number of logical (uncompressed) bytes in the table or\n \ + \ materialized view." name: TOTAL_LOGICAL_BYTES type: INT64 - - description: Number of logical (uncompressed) bytes that are less than 90 days - old + - description: Number of logical (uncompressed) bytes that are younger than 90 days. name: ACTIVE_LOGICAL_BYTES type: INT64 - - description: Number of logical (uncompressed) bytes that are more than 90 days - old + - description: Number of logical (uncompressed) bytes that are older than 90 days. name: LONG_TERM_LOGICAL_BYTES type: INT64 + - description: "Total number of physical bytes for the current storage of the\n\ + \ table across all partitions." + name: CURRENT_PHYSICAL_BYTES + type: INT64 - description: "Total number of physical (compressed) bytes used for storage,\n\ - \ including active, long term, and time travel (deleted or changed data)\ - \ bytes" + \ including active, long-term, and time travel (deleted or changed data)\n\ + \ bytes. Fail-safe (deleted or changed data retained after the time-travel\n\ + \ window) bytes aren't included." name: TOTAL_PHYSICAL_BYTES type: INT64 - - description: "Number of physical (compressed) bytes less than 90 days old, including\n\ - \ time travel (deleted or changed data) bytes" + - description: "Number of physical (compressed) bytes younger than 90 days,\n \ + \ including time travel (deleted or changed data) bytes." name: ACTIVE_PHYSICAL_BYTES type: INT64 - - description: Number of physical (compressed) bytes more than 90 days old + - description: Number of physical (compressed) bytes older than 90 days. name: LONG_TERM_PHYSICAL_BYTES type: INT64 - description: "Number of physical (compressed) bytes used by time travel storage\n\ - \ (deleted or changed data)" + \ (deleted or changed data)." name: TIME_TRAVEL_PHYSICAL_BYTES type: INT64 - - description: "Number of physical (compressed) bytes used by fail-safe storage\n\ - \ (deleted or changed data)" - name: FAIL_SAFE_PHYSICAL_BYTES - type: INT64 - - description: The type of table. For example, `EXTERNAL` or `BASE TABLE` + - description: The most recent time that data was written to the table. + name: STORAGE_LAST_MODIFIED_TIME + type: TIMESTAMP + - description: Indicates whether or not the table is deleted. + name: DELETED + type: BOOLEAN + - description: "The type of table. For example, EXTERNAL or\n BASE TABLE." name: TABLE_TYPE type: STRING + - description: "Number of physical (compressed) bytes used by the fail-safe storage\n\ + \ (deleted or changed data)." + name: FAIL_SAFE_PHYSICAL_BYTES + type: INT64 description: dataset details with related information - name: table_storage + name: information_schema_table_storage version: 2 diff --git a/models/base/google/tables/information_schema_table_storage_by_organization.sql b/models/base/google/tables/information_schema_table_storage_by_organization.sql index 79ee8eb..961aebc 100644 --- a/models/base/google/tables/information_schema_table_storage_by_organization.sql +++ b/models/base/google/tables/information_schema_table_storage_by_organization.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-by-organization -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_BY_ORGANIZATION view, you need the following + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-by-organization -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_BY_ORGANIZATION view, you need the following Identity and Access Management (IAM) permissions for your organization: bigquery.tables.get bigquery.tables.list @@ -12,35 +13,25 @@ roles/bigquery.metadataViewer This schema view is only available to users with defined Google Cloud organizations.For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_BY_ORGANIZATION` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_BY_ORGANIZATION` -{%- endif %} -) -SELECT -project_id, -table_catalog, -project_number, -table_schema, -table_name, -creation_time, -deleted, -storage_last_modified_time, -total_rows, -total_partitions, -total_logical_bytes, -active_logical_bytes, -long_term_logical_bytes, -total_physical_bytes, -active_physical_bytes, -long_term_physical_bytes, -time_travel_physical_bytes, -fail_safe_physical_bytes, -table_type, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT project_id, project_number, table_catalog, table_schema, table_name, creation_time, total_rows, total_partitions, total_logical_bytes, active_logical_bytes, long_term_logical_bytes, current_physical_bytes, total_physical_bytes, active_physical_bytes, long_term_physical_bytes, time_travel_physical_bytes, storage_last_modified_time, deleted, table_type, fail_safe_physical_bytes + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_BY_ORGANIZATION` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT project_id, project_number, table_catalog, table_schema, table_name, creation_time, total_rows, total_partitions, total_logical_bytes, active_logical_bytes, long_term_logical_bytes, current_physical_bytes, total_physical_bytes, active_physical_bytes, long_term_physical_bytes, time_travel_physical_bytes, storage_last_modified_time, deleted, table_type, fail_safe_physical_bytes + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_BY_ORGANIZATION` + + {%- endif %} + ) + SELECT + project_id, project_number, table_catalog, table_schema, table_name, creation_time, total_rows, total_partitions, total_logical_bytes, active_logical_bytes, long_term_logical_bytes, current_physical_bytes, total_physical_bytes, active_physical_bytes, long_term_physical_bytes, time_travel_physical_bytes, storage_last_modified_time, deleted, table_type, fail_safe_physical_bytes, + FROM + base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_table_storage_by_organization.yml b/models/base/google/tables/information_schema_table_storage_by_organization.yml index 210394e..8fbfda8 100644 --- a/models/base/google/tables/information_schema_table_storage_by_organization.yml +++ b/models/base/google/tables/information_schema_table_storage_by_organization.yml @@ -1,73 +1,76 @@ models: - columns: - - description: The project ID of the project that contains the dataset + - description: The project ID of the project that contains the dataset. name: PROJECT_ID type: STRING - - description: The project ID of the project that contains the dataset - name: TABLE_CATALOG - type: STRING - - description: The project number of the project that contains the dataset + - description: The project number of the project that contains the dataset. name: PROJECT_NUMBER type: INT64 + - description: The project ID of the project that contains the dataset. + name: TABLE_CATALOG + type: STRING - description: "The name of the dataset that contains the table or materialized\ - \ view,\n also referred to as the datasetId" + \ view,\n also referred to as the datasetId." name: TABLE_SCHEMA type: STRING - description: "The name of the table or materialized view, also referred to as\ - \ the\n tableId" + \ the\n tableId." name: TABLE_NAME type: STRING - - description: The table's creation time + - description: The creation time of the table. name: CREATION_TIME type: TIMESTAMP - - description: Indicates whether or not the table is deleted - name: DELETED - type: BOOLEAN - - description: The most recent time that data was written to the table. - name: STORAGE_LAST_MODIFIED_TIME - type: TIMESTAMP - - description: The total number of rows in the table or materialized view + - description: The total number of rows in the table or materialized view. name: TOTAL_ROWS type: INT64 - description: "The number of partitions present in the table or materialized view.\n\ \ Unpartitioned tables return 0." name: TOTAL_PARTITIONS type: INT64 - - description: Total number of logical (uncompressed) bytes in the table or materialized - view + - description: "Total number of logical (uncompressed) bytes in the table or\n \ + \ materialized view." name: TOTAL_LOGICAL_BYTES type: INT64 - - description: Number of logical (uncompressed) bytes that are less than 90 days - old + - description: Number of logical (uncompressed) bytes that are younger than 90 days. name: ACTIVE_LOGICAL_BYTES type: INT64 - - description: Number of logical (uncompressed) bytes that are more than 90 days - old + - description: Number of logical (uncompressed) bytes that are older than 90 days. name: LONG_TERM_LOGICAL_BYTES type: INT64 + - description: "Total number of physical bytes for the current storage of the\n\ + \ table across all partitions." + name: CURRENT_PHYSICAL_BYTES + type: INT64 - description: "Total number of physical (compressed) bytes used for storage,\n\ - \ including active, long term, and time travel (deleted or changed data)\ - \ bytes" + \ including active, long-term, and time travel (deleted or changed data)\n\ + \ bytes. Fail-safe (deleted or changed data retained after the time-travel\n\ + \ window) bytes aren't included." name: TOTAL_PHYSICAL_BYTES type: INT64 - - description: "Number of physical (compressed) bytes less than 90 days old, including\n\ - \ time travel (deleted or changed data) bytes" + - description: "Number of physical (compressed) bytes younger than 90 days,\n \ + \ including time travel (deleted or changed data) bytes." name: ACTIVE_PHYSICAL_BYTES type: INT64 - - description: Number of physical (compressed) bytes more than 90 days old + - description: Number of physical (compressed) bytes older than 90 days. name: LONG_TERM_PHYSICAL_BYTES type: INT64 - description: "Number of physical (compressed) bytes used by time travel storage\n\ - \ (deleted or changed data)" + \ (deleted or changed data)." name: TIME_TRAVEL_PHYSICAL_BYTES type: INT64 - - description: "Number of physical (compressed) bytes used by fail-safe storage\n\ - \ (deleted or changed data)" - name: FAIL_SAFE_PHYSICAL_BYTES - type: INT64 - - description: The type of table. For example, `EXTERNAL` or `BASE TABLE` + - description: The most recent time that data was written to the table. + name: STORAGE_LAST_MODIFIED_TIME + type: TIMESTAMP + - description: Indicates whether or not the table is deleted. + name: DELETED + type: BOOLEAN + - description: "The type of table. For example, EXTERNAL or\n BASE TABLE." name: TABLE_TYPE type: STRING + - description: "Number of physical (compressed) bytes used by the fail-safe storage\n\ + \ (deleted or changed data)." + name: FAIL_SAFE_PHYSICAL_BYTES + type: INT64 description: dataset details with related information - name: table_storage_by_organization + name: information_schema_table_storage_by_organization version: 2 diff --git a/models/base/google/tables/information_schema_table_storage_usage_timeline.sql b/models/base/google/tables/information_schema_table_storage_usage_timeline.sql index 4eacb8d..afb1807 100644 --- a/models/base/google/tables/information_schema_table_storage_usage_timeline.sql +++ b/models/base/google/tables/information_schema_table_storage_usage_timeline.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-usage -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_USAGE_TIMELINE view, you need the + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-usage -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_USAGE_TIMELINE view, you need the following Identity and Access Management (IAM) permissions: bigquery.tables.get bigquery.tables.list @@ -11,27 +12,25 @@ roles/bigquery.metadataViewer roles/bigquery.admin For queries with a region qualifier, you must have permissions for the project.For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE` -{%- endif %} -) -SELECT -project_id, -table_catalog, -project_number, -table_schema, -table_name, -billable_total_logical_usage, -billable_active_logical_usage, -billable_long_term_logical_usage, -billable_total_physical_usage, -billable_active_physical_usage, -billable_long_term_physical_usage, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT project_id, table_catalog, project_number, table_schema, table_name, billable_total_logical_usage, billable_active_logical_usage, billable_long_term_logical_usage, billable_total_physical_usage, billable_active_physical_usage, billable_long_term_physical_usage + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT project_id, table_catalog, project_number, table_schema, table_name, billable_total_logical_usage, billable_active_logical_usage, billable_long_term_logical_usage, billable_total_physical_usage, billable_active_physical_usage, billable_long_term_physical_usage + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE` + + {%- endif %} + ) + SELECT + project_id, table_catalog, project_number, table_schema, table_name, billable_total_logical_usage, billable_active_logical_usage, billable_long_term_logical_usage, billable_total_physical_usage, billable_active_physical_usage, billable_long_term_physical_usage, + FROM + base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_table_storage_usage_timeline.yml b/models/base/google/tables/information_schema_table_storage_usage_timeline.yml index 7b7c602..68bbbfe 100644 --- a/models/base/google/tables/information_schema_table_storage_usage_timeline.yml +++ b/models/base/google/tables/information_schema_table_storage_usage_timeline.yml @@ -45,5 +45,5 @@ models: name: BILLABLE_LONG_TERM_PHYSICAL_USAGE type: INT64 description: dataset details with related information - name: table_storage_usage_timeline + name: information_schema_table_storage_usage_timeline version: 2 diff --git a/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql b/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql index 5d0c4ac..85ca079 100644 --- a/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql +++ b/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-usage-by-organization -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_USAGE_TIMELINE_BY_ORGANIZATION view, you + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-table-storage-usage-by-organization -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLE_STORAGE_USAGE_TIMELINE_BY_ORGANIZATION view, you need the following Identity and Access Management (IAM) permissions for your organization: bigquery.tables.get bigquery.tables.list @@ -12,27 +13,25 @@ roles/bigquery.admin This schema view is only available to users with defined Google Cloud organizations.For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE_BY_ORGANIZATION` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE_BY_ORGANIZATION` -{%- endif %} -) -SELECT -project_id, -table_catalog, -project_number, -table_schema, -table_name, -billable_total_logical_usage, -billable_active_logical_usage, -billable_long_term_logical_usage, -billable_total_physical_usage, -billable_active_physical_usage, -billable_long_term_physical_usage, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT project_id, table_catalog, project_number, table_schema, table_name, billable_total_logical_usage, billable_active_logical_usage, billable_long_term_logical_usage, billable_total_physical_usage, billable_active_physical_usage, billable_long_term_physical_usage + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE_BY_ORGANIZATION` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT project_id, table_catalog, project_number, table_schema, table_name, billable_total_logical_usage, billable_active_logical_usage, billable_long_term_logical_usage, billable_total_physical_usage, billable_active_physical_usage, billable_long_term_physical_usage + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLE_STORAGE_USAGE_TIMELINE_BY_ORGANIZATION` + + {%- endif %} + ) + SELECT + project_id, table_catalog, project_number, table_schema, table_name, billable_total_logical_usage, billable_active_logical_usage, billable_long_term_logical_usage, billable_total_physical_usage, billable_active_physical_usage, billable_long_term_physical_usage, + FROM + base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.yml b/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.yml index 97d4e74..1f71685 100644 --- a/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.yml +++ b/models/base/google/tables/information_schema_table_storage_usage_timeline_by_organization.yml @@ -45,5 +45,5 @@ models: name: BILLABLE_LONG_TERM_PHYSICAL_USAGE type: INT64 description: dataset details with related information - name: table_storage_usage_timeline_by_organization + name: information_schema_table_storage_usage_timeline_by_organization version: 2 diff --git a/models/base/google/tables/information_schema_tables.sql b/models/base/google/tables/information_schema_tables.sql index c1e5d60..3c8f99a 100644 --- a/models/base/google/tables/information_schema_tables.sql +++ b/models/base/google/tables/information_schema_tables.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-tables -#} -{# Required role/permissions: To query the INFORMATION_SCHEMA.TABLES view, you need the following + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-tables -#} + {# Required role/permissions: To query the INFORMATION_SCHEMA.TABLES view, you need the following Identity and Access Management (IAM) permissions: bigquery.tables.get bigquery.tables.list @@ -12,35 +13,25 @@ roles/bigquery.dataViewer roles/bigquery.metadataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLES` -{%- endif %} -) -SELECT -table_catalog, -table_schema, -table_name, -table_type, -is_insertable_into, -is_typed, -creation_time, -base_table_catalog, -base_table_schema, -base_table_name, -snapshot_time_ms, -replica_source_catalog, -replica_source_schema, -replica_source_name, -replication_status, -replication_error, -ddl, -default_collation_name, -upsert_stream_apply_watermark, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT table_catalog, table_schema, table_name, table_type, is_insertable_into, is_typed, creation_time, base_table_catalog, base_table_schema, base_table_name, snapshot_time_ms, replica_source_catalog, replica_source_schema, replica_source_name, replication_status, replication_error, ddl, default_collation_name, upsert_stream_apply_watermark + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLES` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT table_catalog, table_schema, table_name, table_type, is_insertable_into, is_typed, creation_time, base_table_catalog, base_table_schema, base_table_name, snapshot_time_ms, replica_source_catalog, replica_source_schema, replica_source_name, replication_status, replication_error, ddl, default_collation_name, upsert_stream_apply_watermark + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`TABLES` + + {%- endif %} + ) + SELECT + table_catalog, table_schema, table_name, table_type, is_insertable_into, is_typed, creation_time, base_table_catalog, base_table_schema, base_table_name, snapshot_time_ms, replica_source_catalog, replica_source_schema, replica_source_name, replication_status, replication_error, ddl, default_collation_name, upsert_stream_apply_watermark, + FROM + base + \ No newline at end of file diff --git a/models/base/google/tables/information_schema_tables.yml b/models/base/google/tables/information_schema_tables.yml index 517cb07..d326cd1 100644 --- a/models/base/google/tables/information_schema_tables.yml +++ b/models/base/google/tables/information_schema_tables.yml @@ -94,5 +94,5 @@ models: name: upsert_stream_apply_watermark type: TIMESTAMP description: dataset details with related information - name: tables + name: information_schema_tables version: 2 diff --git a/models/base/google/vector_indexes/information_schema_vector_index_columns.sql b/models/base/google/vector_indexes/information_schema_vector_index_columns.sql index 2136a71..a47a9b9 100644 --- a/models/base/google/vector_indexes/information_schema_vector_index_columns.sql +++ b/models/base/google/vector_indexes/information_schema_vector_index_columns.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-index-columns -#} -{# Required role/permissions: To see vector index metadata, you need the + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-index-columns -#} + {# Required role/permissions: To see vector index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) permission on the table with the index. Each of the following predefined IAM roles includes at least one of these permissions: @@ -11,22 +12,25 @@ roles/bigquery.metadataViewer roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_COLUMNS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_COLUMNS` -{%- endif %} -) -SELECT -index_catalog, -index_schema, -table_name, -index_name, -index_column_name, -index_field_path, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT index_catalog, index_schema, table_name, index_name, index_column_name, index_field_path + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_COLUMNS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT index_catalog, index_schema, table_name, index_name, index_column_name, index_field_path + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_COLUMNS` + + {%- endif %} + ) + SELECT + index_catalog, index_schema, table_name, index_name, index_column_name, index_field_path, + FROM + base + \ No newline at end of file diff --git a/models/base/google/vector_indexes/information_schema_vector_index_columns.yml b/models/base/google/vector_indexes/information_schema_vector_index_columns.yml index b06a08f..bd0275e 100644 --- a/models/base/google/vector_indexes/information_schema_vector_index_columns.yml +++ b/models/base/google/vector_indexes/information_schema_vector_index_columns.yml @@ -20,5 +20,5 @@ models: name: index_field_path type: STRING description: dataset details with related information - name: vector_index_columns + name: information_schema_vector_index_columns version: 2 diff --git a/models/base/google/vector_indexes/information_schema_vector_index_options.sql b/models/base/google/vector_indexes/information_schema_vector_index_options.sql index 37c54de..d984277 100644 --- a/models/base/google/vector_indexes/information_schema_vector_index_options.sql +++ b/models/base/google/vector_indexes/information_schema_vector_index_options.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-index-options -#} -{# Required role/permissions: To see vector index metadata, you need the + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-index-options -#} + {# Required role/permissions: To see vector index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) permission on the table with the index. Each of the following predefined IAM roles includes at least one of these permissions: @@ -11,23 +12,25 @@ roles/bigquery.metadataViewer roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_OPTIONS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_OPTIONS` -{%- endif %} -) -SELECT -index_catalog, -index_schema, -table_name, -index_name, -option_name, -option_type, -option_value, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT index_catalog, index_schema, table_name, index_name, option_name, option_type, option_value + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_OPTIONS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT index_catalog, index_schema, table_name, index_name, option_name, option_type, option_value + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEX_OPTIONS` + + {%- endif %} + ) + SELECT + index_catalog, index_schema, table_name, index_name, option_name, option_type, option_value, + FROM + base + \ No newline at end of file diff --git a/models/base/google/vector_indexes/information_schema_vector_index_options.yml b/models/base/google/vector_indexes/information_schema_vector_index_options.yml index 7a30aa6..1cbf4c8 100644 --- a/models/base/google/vector_indexes/information_schema_vector_index_options.yml +++ b/models/base/google/vector_indexes/information_schema_vector_index_options.yml @@ -23,5 +23,5 @@ models: name: option_value type: STRING description: dataset details with related information - name: vector_index_options + name: information_schema_vector_index_options version: 2 diff --git a/models/base/google/vector_indexes/information_schema_vector_indexes.sql b/models/base/google/vector_indexes/information_schema_vector_indexes.sql index da23f90..bdad8c1 100644 --- a/models/base/google/vector_indexes/information_schema_vector_indexes.sql +++ b/models/base/google/vector_indexes/information_schema_vector_indexes.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-indexes -#} -{# Required role/permissions: To see vector index metadata, you need the + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-vector-indexes -#} + {# Required role/permissions: To see vector index metadata, you need the bigquery.tables.get or bigquery.tables.list Identity and Access Management (IAM) permission on the table with the index. Each of the following predefined IAM roles includes at least one of these permissions: @@ -11,31 +12,25 @@ roles/bigquery.metadataViewer roles/bigquery.user For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEXES` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEXES` -{%- endif %} -) -SELECT -index_catalog, -index_schema, -table_name, -index_name, -index_status, -creation_time, -last_modification_time, -last_refresh_time, -disable_time, -disable_reason, -ddl, -coverage_percentage, -unindexed_row_count, -total_logical_bytes, -total_storage_bytes, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT index_catalog, index_schema, table_name, index_name, index_status, creation_time, last_modification_time, last_refresh_time, disable_time, disable_reason, ddl, coverage_percentage, unindexed_row_count, total_logical_bytes, total_storage_bytes + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEXES` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT index_catalog, index_schema, table_name, index_name, index_status, creation_time, last_modification_time, last_refresh_time, disable_time, disable_reason, ddl, coverage_percentage, unindexed_row_count, total_logical_bytes, total_storage_bytes + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VECTOR_INDEXES` + + {%- endif %} + ) + SELECT + index_catalog, index_schema, table_name, index_name, index_status, creation_time, last_modification_time, last_refresh_time, disable_time, disable_reason, ddl, coverage_percentage, unindexed_row_count, total_logical_bytes, total_storage_bytes, + FROM + base + \ No newline at end of file diff --git a/models/base/google/vector_indexes/information_schema_vector_indexes.yml b/models/base/google/vector_indexes/information_schema_vector_indexes.yml index d806414..8ac9b66 100644 --- a/models/base/google/vector_indexes/information_schema_vector_indexes.yml +++ b/models/base/google/vector_indexes/information_schema_vector_indexes.yml @@ -67,5 +67,5 @@ models: name: total_storage_bytes type: INTEGER description: dataset details with related information - name: vector_indexes + name: information_schema_vector_indexes version: 2 diff --git a/models/base/google/views/information_schema_materialized_views.sql b/models/base/google/views/information_schema_materialized_views.sql index cb1f49d..bafdb97 100644 --- a/models/base/google/views/information_schema_materialized_views.sql +++ b/models/base/google/views/information_schema_materialized_views.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-materialized-views -#} -{# Required role/permissions: + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-materialized-views -#} + {# Required role/permissions: To get the permissions that you need to query the INFORMATION_SCHEMA.MATERIALIZED_VIEWS view, @@ -39,22 +40,25 @@ The following permissions are required to query the INFORMATION_SCHEMA.MATERIALI with custom roles or other predefined roles. Access control with IAM -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`MATERIALIZED_VIEWS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`MATERIALIZED_VIEWS` -{%- endif %} -) -SELECT -table_catalog, -table_schema, -table_name, -last_refresh_time, -refresh_watermark, -last_refresh_status, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT table_catalog, table_schema, table_name, last_refresh_time, refresh_watermark, last_refresh_status + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`MATERIALIZED_VIEWS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT table_catalog, table_schema, table_name, last_refresh_time, refresh_watermark, last_refresh_status + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`MATERIALIZED_VIEWS` + + {%- endif %} + ) + SELECT + table_catalog, table_schema, table_name, last_refresh_time, refresh_watermark, last_refresh_status, + FROM + base + \ No newline at end of file diff --git a/models/base/google/views/information_schema_materialized_views.yml b/models/base/google/views/information_schema_materialized_views.yml index 0c5568c..f3eed8d 100644 --- a/models/base/google/views/information_schema_materialized_views.yml +++ b/models/base/google/views/information_schema_materialized_views.yml @@ -25,5 +25,5 @@ models: name: LAST_REFRESH_STATUS type: RECORD description: dataset details with related information - name: materialized_views + name: information_schema_materialized_views version: 2 diff --git a/models/base/google/views/information_schema_views.sql b/models/base/google/views/information_schema_views.sql index 50c5871..b3705d6 100644 --- a/models/base/google/views/information_schema_views.sql +++ b/models/base/google/views/information_schema_views.sql @@ -1,5 +1,6 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-views -#} -{# Required role/permissions: To get view metadata, you need the following Identity and Access Management (IAM) + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-views -#} + {# Required role/permissions: To get view metadata, you need the following Identity and Access Management (IAM) permissions: bigquery.tables.get bigquery.tables.list @@ -11,22 +12,25 @@ roles/bigquery.metadataViewer roles/bigquery.dataViewer For more information about BigQuery permissions, see Access control with IAM. -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VIEWS` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VIEWS` -{%- endif %} -) -SELECT -table_catalog, -table_schema, -table_name, -view_definition, -check_option, -use_standard_sql, -FROM - base + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT table_catalog, table_schema, table_name, view_definition, check_option, use_standard_sql + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VIEWS` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT table_catalog, table_schema, table_name, view_definition, check_option, use_standard_sql + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`VIEWS` + + {%- endif %} + ) + SELECT + table_catalog, table_schema, table_name, view_definition, check_option, use_standard_sql, + FROM + base + \ No newline at end of file diff --git a/models/base/google/views/information_schema_views.yml b/models/base/google/views/information_schema_views.yml index e6b7908..3f38d94 100644 --- a/models/base/google/views/information_schema_views.yml +++ b/models/base/google/views/information_schema_views.yml @@ -21,5 +21,5 @@ models: name: USE_STANDARD_SQL type: STRING description: dataset details with related information - name: views + name: information_schema_views version: 2 diff --git a/models/base/google/write_api/information_schema_write_api_timeline.sql b/models/base/google/write_api/information_schema_write_api_timeline.sql index 7ad12b3..13a9879 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline.sql +++ b/models/base/google/write_api/information_schema_write_api_timeline.sql @@ -1,24 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE` -{%- endif %} -) -SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -stream_type, -error_code, -total_requests, -total_rows, -total_input_bytes, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE` + + {%- endif %} + ) + SELECT + start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes, + FROM + base + \ No newline at end of file diff --git a/models/base/google/write_api/information_schema_write_api_timeline.yml b/models/base/google/write_api/information_schema_write_api_timeline.yml index 48a4b15..b68d29e 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline.yml +++ b/models/base/google/write_api/information_schema_write_api_timeline.yml @@ -35,5 +35,5 @@ models: name: total_input_bytes type: INTEGER description: dataset details with related information - name: write_api_timeline + name: information_schema_write_api_timeline version: 2 diff --git a/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql b/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql index fee8cf3..fa36e64 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql +++ b/models/base/google/write_api/information_schema_write_api_timeline_by_folder.sql @@ -1,24 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api-by-folder -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_FOLDER` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_FOLDER` -{%- endif %} -) -SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -stream_type, -error_code, -total_requests, -total_rows, -total_input_bytes, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api-by-folder -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_FOLDER` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_FOLDER` + + {%- endif %} + ) + SELECT + start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes, + FROM + base + \ No newline at end of file diff --git a/models/base/google/write_api/information_schema_write_api_timeline_by_folder.yml b/models/base/google/write_api/information_schema_write_api_timeline_by_folder.yml index 6cb34e3..502dfa7 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline_by_folder.yml +++ b/models/base/google/write_api/information_schema_write_api_timeline_by_folder.yml @@ -35,5 +35,5 @@ models: name: total_input_bytes type: INTEGER description: dataset details with related information - name: write_api_timeline_by_folder + name: information_schema_write_api_timeline_by_folder version: 2 diff --git a/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql b/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql index 315eb51..0ad5909 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql +++ b/models/base/google/write_api/information_schema_write_api_timeline_by_organization.sql @@ -1,24 +1,24 @@ -{# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api-by-organization -#} -WITH base AS ( -{% if project_list()|length > 0 -%} - {% for project in project_list() -%} - SELECT * FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_ORGANIZATION` - {% if not loop.last %}UNION ALL{% endif %} - {% endfor %} -{%- else %} - SELECT * FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_ORGANIZATION` -{%- endif %} -) -SELECT -start_timestamp, -project_id, -project_number, -dataset_id, -table_id, -stream_type, -error_code, -total_requests, -total_rows, -total_input_bytes, -FROM - base + + {# More details about base table in https://cloud.google.com/bigquery/docs/information-schema-write-api-by-organization -#} + + WITH base AS ( + {% if project_list()|length > 0 -%} + {% for project in project_list() -%} + + SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes + FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_ORGANIZATION` + + {% if not loop.last %}UNION ALL{% endif %} + {% endfor %} + {%- else %} + + SELECT start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes + FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`WRITE_API_TIMELINE_BY_ORGANIZATION` + + {%- endif %} + ) + SELECT + start_timestamp, project_id, project_number, dataset_id, table_id, stream_type, error_code, total_requests, total_rows, total_input_bytes, + FROM + base + \ No newline at end of file diff --git a/models/base/google/write_api/information_schema_write_api_timeline_by_organization.yml b/models/base/google/write_api/information_schema_write_api_timeline_by_organization.yml index 32715d3..94a505f 100644 --- a/models/base/google/write_api/information_schema_write_api_timeline_by_organization.yml +++ b/models/base/google/write_api/information_schema_write_api_timeline_by_organization.yml @@ -35,5 +35,5 @@ models: name: total_input_bytes type: INTEGER description: dataset details with related information - name: write_api_timeline_by_organization + name: information_schema_write_api_timeline_by_organization version: 2 diff --git a/models/base/table_and_storage_with_cost.sql b/models/base/table_and_storage_with_cost.sql index 7928c8c..ea5070e 100644 --- a/models/base/table_and_storage_with_cost.sql +++ b/models/base/table_and_storage_with_cost.sql @@ -8,9 +8,9 @@ SELECT t.is_insertable_into, t.is_typed, t.ddl, - t.base_table_catalog AS project_id, - t.base_table_schema AS dataset_id, - t.base_table_name AS table_id, + t.table_catalog AS project_id, + t.table_schema AS dataset_id, + t.table_name AS table_id, t.default_collation_name, s.project_number, s.creation_time, @@ -19,12 +19,13 @@ SELECT s.total_rows, s.total_partitions, s.total_logical_bytes, - s.active_logical_bytes, - s.long_term_logical_bytes, + IF(deleted=false, s.active_logical_bytes, 0) active_logical_bytes, + IF(deleted=false, s.long_term_logical_bytes, 0) long_term_logical_bytes, s.total_physical_bytes, s.active_physical_bytes, s.long_term_physical_bytes, s.time_travel_physical_bytes, + s.fail_safe_physical_bytes, s.table_type FROM {{ ref('information_schema_tables') }} t INNER JOIN {{ ref('information_schema_table_storage') }} AS s ON @@ -34,17 +35,34 @@ SELECT base_with_enriched_fields AS ( SELECT *, + -- Logical storage costs active_logical_bytes / POW(1024, 3) * {{ var('active_logical_storage_gb_price') }} AS active_logical_bytes_cost, long_term_logical_bytes / POW(1024, 3) * {{ var('long_term_logical_storage_gb_price') }} AS long_term_logical_bytes_cost, + -- Physical storage costs active_physical_bytes / POW(1024, 3) * {{ var('active_physical_storage_gb_price') }} AS active_physical_bytes_cost, + time_travel_physical_bytes / POW(1024, 3) * {{ var('active_physical_storage_gb_price') }} AS time_travel_physical_bytes_cost, + fail_safe_physical_bytes / POW(1024, 3) * {{ var('active_physical_storage_gb_price') }} AS fail_safe_physical_bytes_cost, long_term_physical_bytes / POW(1024, 3) * {{ var('long_term_physical_storage_gb_price') }} AS long_term_physical_bytes_cost, - time_travel_physical_bytes / POW(1024, 3) * {{ var('active_physical_storage_gb_price') }} AS time_travel_physical_bytes_cost FROM base -) - +), +base_with_forecast AS ( SELECT *, {{ sharded_table_merger("table_id") }} factored_table_id, - total_logical_bytes / POW(1024, 4) AS total_tb, - active_logical_bytes_cost + long_term_logical_bytes_cost + active_physical_bytes_cost + long_term_physical_bytes_cost + time_travel_physical_bytes_cost AS storage_cost + total_logical_bytes / POW(1024, 4) AS total_logical_tb, + total_physical_bytes / POW(1024, 4) AS total_physical_tb, + (active_logical_bytes_cost + long_term_logical_bytes_cost) AS logical_cost_monthly_forecast, + (active_physical_bytes_cost + long_term_physical_bytes_cost) AS physical_cost_monthly_forecast, + ((active_physical_bytes_cost - time_travel_physical_bytes_cost - fail_safe_physical_bytes_cost) + long_term_physical_bytes_cost) AS physical_cost_monthly_forecast_with_zero_time_travel_and_fail_safe, + SAFE_DIVIDE(time_travel_physical_bytes, active_physical_bytes) as time_travel_per_active_byte_ratio, + SAFE_DIVIDE(fail_safe_physical_bytes, active_physical_bytes) as fail_safe_per_active_byte_ratio, + SAFE_DIVIDE(active_logical_bytes, (active_physical_bytes - time_travel_physical_bytes)) AS active_compression_ratio, + SAFE_DIVIDE(long_term_logical_bytes, long_term_physical_bytes) AS long_term_compression_ratio, + FROM base_with_enriched_fields +) +SELECT + *, + (logical_cost_monthly_forecast - physical_cost_monthly_forecast) > 0 AS prefer_physical_pricing_model, + ABS(logical_cost_monthly_forecast - physical_cost_monthly_forecast) AS storage_pricing_model_difference +FROM base_with_forecast diff --git a/models/base/table_and_storage_with_cost.yml b/models/base/table_and_storage_with_cost.yml index 3b64a25..04fe9d6 100644 --- a/models/base/table_and_storage_with_cost.yml +++ b/models/base/table_and_storage_with_cost.yml @@ -121,3 +121,51 @@ models: - name: time_travel_physical_bytes_cost description: The cost of the time travel physical bytes cost part of the storage in the reference currency (default values are in dollars). This time travel data is the historical data (that has been deleted or changed). The default time travel window is 7 days. The physical bytes are the data counted as compressed bytes. type: FLOAT64 + + - name: fail_safe_physical_bytes_cost + description: The cost of the fail safe physical bytes cost part of the storage in the reference currency (default values are in dollars). This time travel data is the historical data (that has been deleted or changed). The default time travel window is 7 days. The physical bytes are the data counted as compressed bytes. + type: FLOAT64 + + - name: factored_table_id + description: The factored table id + type: STRING + + - name: total_logical_tb + description: Total number of logical (uncompressed) terabytes in the table or materialized view + type: FLOAT64 + + - name: total_physical_tb + description: Total number of physical (compressed) terabytes used for storage, including active, long term, and time travel (deleted or changed data) terabytes + type: FLOAT64 + + - name: logical_cost_monthly_forecast + description: The forecasted cost of the logical bytes cost part of the storage in the reference currency (default values are in dollars). The logical bytes are the data counted as raw (uncompressed) bytes. + type: FLOAT64 + + - name: physical_cost_monthly_forecast + description: The forecasted cost of the physical bytes cost part of the storage in the reference currency (default values are in dollars). The physical bytes are the data counted as compressed bytes. + type: FLOAT64 + + - name: physical_cost_monthly_forecast_with_zero_time_travel_and_fail_safe + description: The forecasted cost of the physical bytes cost part of the storage in the reference currency (default values are in dollars) with zero time travel and fail safe. The physical bytes are the data counted as compressed bytes. + type: FLOAT64 + + - name: time_travel_per_active_byte_ratio + description: The ratio of time travel bytes to active bytes + type: FLOAT64 + + - name: fail_safe_per_active_byte_ratio + description: The ratio of fail safe bytes to active bytes + type: FLOAT64 + + - name: active_compression_ratio + description: The ratio of active physical bytes to active logical bytes + type: FLOAT64 + + - name: long_term_compression_ratio + description: The ratio of long term physical bytes to long term logical bytes + type: FLOAT64 + + - name: storage_pricing_model_difference + description: The difference between the storage pricing model and the default pricing model + type: STRING diff --git a/models/storage/datamart/dataset_with_better_pricing_on_logical_billing_model.sql b/models/storage/datamart/dataset_with_better_pricing_on_logical_billing_model.sql new file mode 100644 index 0000000..749a716 --- /dev/null +++ b/models/storage/datamart/dataset_with_better_pricing_on_logical_billing_model.sql @@ -0,0 +1,25 @@ +{{ + config( + materialized='table', + ) +}} +WITH storage_cost AS ( + SELECT + project_id, + dataset_id, + SUM(total_logical_tb) AS total_logical_tb, + SUM(total_physical_tb) AS total_physical_tb, + SUM(logical_cost_monthly_forecast) AS logical_cost_monthly_forecast, + SUM(physical_cost_monthly_forecast) AS physical_cost_monthly_forecast, + SUM(storage_pricing_model_difference) AS storage_pricing_model_difference + FROM {{ ref('table_and_storage_with_cost') }} + WHERE total_physical_bytes > 0 + AND table_type = 'BASE TABLE' + AND NOT prefer_physical_pricing_model + GROUP BY ALL +) +SELECT + * +FROM storage_cost +ORDER BY storage_pricing_model_difference DESC +LIMIT {{ var('output_limit_size') }} diff --git a/models/storage/datamart/dataset_with_better_pricing_on_physical_billing_model.sql b/models/storage/datamart/dataset_with_better_pricing_on_physical_billing_model.sql new file mode 100644 index 0000000..54bba40 --- /dev/null +++ b/models/storage/datamart/dataset_with_better_pricing_on_physical_billing_model.sql @@ -0,0 +1,25 @@ +{{ + config( + materialized='table', + ) +}} +WITH storage_cost AS ( + SELECT + project_id, + dataset_id, + SUM(total_logical_tb) AS total_logical_tb, + SUM(total_physical_tb) AS total_physical_tb, + SUM(logical_cost_monthly_forecast) AS logical_cost_monthly_forecast, + SUM(physical_cost_monthly_forecast) AS physical_cost_monthly_forecast, + SUM(storage_pricing_model_difference) AS storage_pricing_model_difference + FROM {{ ref('table_and_storage_with_cost') }} + WHERE total_physical_bytes > 0 + AND table_type = 'BASE TABLE' + AND prefer_physical_pricing_model + GROUP BY ALL +) +SELECT + * +FROM storage_cost +ORDER BY storage_pricing_model_difference DESC +LIMIT {{ var('output_limit_size') }} diff --git a/models/storage/datamart/most_expensive_tables.sql b/models/storage/datamart/most_expensive_tables.sql index b313cb4..5634f79 100644 --- a/models/storage/datamart/most_expensive_tables.sql +++ b/models/storage/datamart/most_expensive_tables.sql @@ -6,5 +6,9 @@ SELECT * FROM {{ ref('table_and_storage_with_cost') }} -ORDER BY storage_cost DESC +{%- if var('prefer_physical_pricing_model') %} +ORDER BY physical_cost_monthly_forecast DESC +{%- else %} +ORDER BY logical_cost_monthly_forecast DESC +{%- endif %} LIMIT {{ var('output_limit_size') }} diff --git a/models/storage/datamart/table_with_better_pricing_on_logical_billing_model.sql b/models/storage/datamart/table_with_better_pricing_on_logical_billing_model.sql new file mode 100644 index 0000000..e8719ca --- /dev/null +++ b/models/storage/datamart/table_with_better_pricing_on_logical_billing_model.sql @@ -0,0 +1,25 @@ +{{ + config( + materialized='table', + ) +}} +WITH storage_cost AS ( + SELECT + project_id, + dataset_id, + table_id, + total_logical_tb, + total_physical_tb, + logical_cost_monthly_forecast, + physical_cost_monthly_forecast, + storage_pricing_model_difference + FROM {{ ref('table_and_storage_with_cost') }} + WHERE total_physical_bytes > 0 + AND table_type = 'BASE TABLE' + AND NOT prefer_physical_pricing_model +) +SELECT + * +FROM storage_cost +ORDER BY storage_pricing_model_difference DESC +LIMIT {{ var('output_limit_size') }} diff --git a/models/storage/datamart/table_with_better_pricing_on_physical_billing_model.sql b/models/storage/datamart/table_with_better_pricing_on_physical_billing_model.sql new file mode 100644 index 0000000..92688e1 --- /dev/null +++ b/models/storage/datamart/table_with_better_pricing_on_physical_billing_model.sql @@ -0,0 +1,25 @@ +{{ + config( + materialized='table', + ) +}} +WITH storage_cost AS ( + SELECT + project_id, + dataset_id, + table_id, + total_logical_tb, + total_physical_tb, + logical_cost_monthly_forecast, + physical_cost_monthly_forecast, + storage_pricing_model_difference + FROM {{ ref('table_and_storage_with_cost') }} + WHERE total_physical_bytes > 0 + AND table_type = 'BASE TABLE' + AND prefer_physical_pricing_model +) +SELECT + * +FROM storage_cost +ORDER BY storage_pricing_model_difference DESC +LIMIT {{ var('output_limit_size') }} diff --git a/models/storage/datamart/unused_tables.sql b/models/storage/datamart/unused_tables.sql index 209fcd2..e2a1536 100644 --- a/models/storage/datamart/unused_tables.sql +++ b/models/storage/datamart/unused_tables.sql @@ -3,8 +3,17 @@ materialized='table', ) }} +WITH last_used_date AS ( + SELECT + project_id, + dataset_id, + table_id, + MAX(day) AS last_used_date + FROM {{ ref('table_reference_incremental') }} AS ut + GROUP BY ALL +) SELECT - ts.* + ts.*, + last_used_date FROM {{ ref('table_and_storage_with_cost') }} AS ts -LEFT JOIN {{ ref('table_reference_incremental') }} AS ut USING (project_id, dataset_id, table_id) -WHERE ut.table_id IS NULL +LEFT JOIN last_used_date USING (project_id, dataset_id, table_id) diff --git a/models/storage/intermediate/table_reference_incremental.yml b/models/storage/intermediate/table_reference_incremental.yml index 7b4f25e..f060733 100644 --- a/models/storage/intermediate/table_reference_incremental.yml +++ b/models/storage/intermediate/table_reference_incremental.yml @@ -2,25 +2,25 @@ version: 2 models: - name: table_reference_incremental - description: "" + description: This table is storing the reference count of each table in actual BigQuery queries. columns: - name: day data_type: timestamp - description: "" + description: day of the creation_time - name: project_id data_type: string - description: "" + description: GCP project id from the referenced table - name: dataset_id data_type: string - description: "" + description: dataset id from the referenced table - name: table_id data_type: string - description: "" + description: reference table - name: reference_count data_type: int64 - description: "" + description: number of references to the table for the given day