Merge pull request #9 from bqbooster/google_model_update

Update google models, add new dataset with costs and clean up
bqbooster · Aug 20, 2024 · b6b683f · b6b683f
2 parents 83477c8 + 51852be
commit b6b683f
Show file tree

Hide file tree

Showing 29 changed files with 683 additions and 60 deletions.
diff --git a/.changes/v0.5.4.md b/.changes/v0.5.4.md
@@ -0,0 +1,9 @@
+## dbt-bigquery-monitoring v0.5.4 - August 19, 2024
+
+### Features
+
+- Add a model with properties ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0))
+
+### Contributors
+- [@Kayrnt](https://github.com/Kayrnt) ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0))
+
diff --git a/.changes/v0.6.0.md b/.changes/v0.6.0.md
@@ -0,0 +1,9 @@
+## dbt-bigquery-monitoring v0.6.0 - August 20, 2024
+
+### Breaking Changes
+
+- Update google models, add new dataset with costs and clean up ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0))
+
+### Contributors
+- [@Kayrnt](https://github.com/Kayrnt) ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0))
+
diff --git a/.changie.yaml b/.changie.yaml
@@ -10,11 +10,15 @@ kindFormat: '### {{.Kind}}'
 changeFormat: |-
   {{- $IssueList := list }}
   {{- $changes := splitList " " $.Custom.Issue }}
-  {{- range $issueNbr := $changes }}
-    {{- $changeLink := "[#nbr](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/nbr)" | replace "nbr" $issueNbr }}
-    {{- $IssueList = append $IssueList $changeLink  }}
-  {{- end -}}
-  - {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}})
+  {{- if ne (len $changes) 0 }}
+    {{- range $issueNbr := $changes }}
+      {{- $changeLink := "[#nbr](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/nbr)" | replace "nbr" $issueNbr }}
+      {{- $IssueList = append $IssueList $changeLink }}
+    {{- end }}
+    - {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}})
+  {{- else }}
+    - {{.Body}}
+  {{- end }}
 
 kinds:
   - label: Breaking Changes
@@ -38,9 +42,9 @@ custom:
   type: string
   minLength: 3
 - key: Issue
-  label: GitHub Issue Number (separated by a single space if multiple)
+  label: GitHub Issue Number (optional, separated by a single space if multiple)
   type: string
-  minLength: 1
+  minLength: 0
 
 footerFormat: |
   {{- $contributorDict := dict }}
@@ -53,30 +57,30 @@ footerFormat: |
       {{- $authorLower := lower $author }}
       {{- /* we only want to include non-core team contributors */}}
       {{- if not (has $authorLower $maintainers)}}
-        {{- $changeList := splitList " " $change.Custom.Author }}
-          {{- $IssueList := list }}
-          {{- $changeLink := $change.Kind }}
-          {{- $changes := splitList " " $change.Custom.Issue }}
+        {{- $IssueList := list }}
+        {{- $changes := splitList " " $change.Custom.Issue }}
+        {{- if ne (len $changes) 0 }}
           {{- range $issueNbr := $changes }}
             {{- $changeLink := "[#nbr](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/nbr)" | replace "nbr" $issueNbr }}
-            {{- $IssueList = append $IssueList $changeLink  }}
-          {{- end }}
-          {{- /* check if this contributor has other changes associated with them already */}}
-          {{- if hasKey $contributorDict $author }}
-            {{- $contributionList := get $contributorDict $author }}
-            {{- $contributionList = concat $contributionList $IssueList  }}
-            {{- $contributorDict := set $contributorDict $author $contributionList }}
-          {{- else }}
-            {{- $contributionList := $IssueList }}
-            {{- $contributorDict := set $contributorDict $author $contributionList }}
+            {{- $IssueList = append $IssueList $changeLink }}
           {{- end }}
-        {{- end}}
-    {{- end}}
+        {{- end }}
+        {{- /* check if this contributor has other changes associated with them already */}}
+        {{- if hasKey $contributorDict $author }}
+          {{- $contributionList := get $contributorDict $author }}
+          {{- $contributionList = concat $contributionList $IssueList }}
+          {{- $contributorDict = set $contributorDict $author $contributionList }}
+        {{- else }}
+          {{- $contributionList := $IssueList }}
+          {{- $contributorDict = set $contributorDict $author $contributionList }}
+        {{- end }}
+      {{- end }}
+    {{- end }}
   {{- end }}
   {{- /* no indentation here for formatting so the final markdown doesn't have unneeded indentations */}}
   {{- if $contributorDict}}
   ### Contributors
   {{- range $k,$v := $contributorDict }}
-  - [@{{$k}}](https://github.com/{{$k}}) ({{ range $index, $element := $v }}{{if $index}}, {{end}}{{$element}}{{end}})
+  - [@{{$k}}](https://github.com/{{$k}}){{ if $v }} ({{ range $index, $element := $v }}{{if $index}}, {{end}}{{$element}}{{end}}){{ end }}
   {{- end }}
   {{- end }}
diff --git a/.github/workflows/changie_generate_release_pr.yml b/.github/workflows/changie_generate_release_pr.yml
@@ -0,0 +1,35 @@
+on:
+  workflow_dispatch:
+
+jobs:
+  generate-pr:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Batch changes
+      uses: miniscruff/changie-action@v2
+      with:
+        version: latest
+        args: batch auto
+
+    - name: Merge changes
+      uses: miniscruff/changie-action@v2
+      with:
+        version: latest
+        args: merge
+
+    - name: Get the latest version
+      id: latest
+      uses: miniscruff/changie-action@v2
+      with:
+        version: latest
+        args: latest
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v4
+      with:
+        title: Release ${{ steps.latest.outputs.output }}
+        branch: release/${{ steps.latest.outputs.output }}
+        commit-message: Release ${{ steps.latest.outputs.output }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,32 @@
+name: release
+
+on:
+  push:
+    branches: [ main ]
+    paths: [ CHANGELOG.md ]
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Get the latest version
+      id: latest
+      uses: miniscruff/changie-action@v2
+      with:
+        version: latest
+        args: latest
+
+    - name: Release
+      uses: softprops/action-gh-release@v2
+      with:
+        body_path: ".changes/${{ steps.latest.outputs.output }}.md"
+        tag_name: "${{ steps.latest.outputs.output }}"
+
+    - name: Release
+      uses: softprops/action-gh-release@v2
+      with:
+        body: View full release notes in the [Changelog](https://github.com/bqbooster/dbt-bigquery-monitoring/blob/main/CHANGELOG.md)
+        tag_name: v2 # currently no way of finding this in changie directly
diff --git a/.gitignore b/.gitignore
@@ -1,11 +1,19 @@
+# IDE
 .idea/
+.vscode/
+# cache
+.cache
 # standard dbt ignore
 target/
 dbt_modules/
 dbt_packages/
 logs/
 /.user.yml
-documenation_parser/html_content.html
 # ignore any __pycache__ directories
 __pycache__/
+# ignore temporary documentation parser files
 documentation_parser/tmp_html_content.html
+documentation_parser/output/
+# reports
+reports/
+
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
+## dbt-bigquery-monitoring v0.6.0 - August 20, 2024
+
+### Breaking Changes
+
+- Update google models, add new dataset with costs and clean up ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0))
+
+### Contributors
+- [@Kayrnt](https://github.com/Kayrnt) ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0))
+
+
 ## dbt-bigquery-monitoring v0.5.4 - August 19, 2024
 
 ### Features

diff --git a/documentation_parser/config.py b/documentation_parser/config.py
@@ -102,6 +102,19 @@
         "dir": "jobs_timeline",
         "url": "https://cloud.google.com/bigquery/docs/information-schema-jobs-timeline-by-organization",
     },
+    # recommendations and insights
+    "insights": {
+        "dir": "recommendations_and_insights",
+        "url": "https://cloud.google.com/bigquery/docs/information-schema-insights",
+    },
+    "recommendations": {
+        "dir": "recommendations_and_insights",
+        "url": "https://cloud.google.com/bigquery/docs/information-schema-recommendations",
+    },
+    "recommendations_by_organization": {
+        "dir": "recommendations_and_insights",
+        "url": "https://cloud.google.com/bigquery/docs/information-schema-recommendations-by-org",
+    },
     # reservations
     "assignments": {
         "dir": "reservations",

diff --git a/models/base/google/access_control/information_schema_object_privileges.yml b/models/base/google/access_control/information_schema_object_privileges.yml
@@ -3,11 +3,11 @@ models:
   - description: The project ID of the project that contains the resource.
     name: OBJECT_CATALOG
     type: STRING
-  - description: The name of the dataset that contains the resource. This is null
-      for dataset resource types.
+  - description: "The name of the dataset that contains the resource. This is\n  \
+      \      NULL if the resource itself is a dataset."
     name: OBJECT_SCHEMA
     type: STRING
-  - description: The name of the table, view or dataset the policy applies to.
+  - description: The name of the table, view, or dataset the policy applies to.
     name: OBJECT_NAME
     type: STRING
   - description: The resource type, such as SCHEMA (dataset), TABLE, VIEW, and EXTERNAL.

diff --git a/models/base/google/configuration/information_schema_organization_options_changes.sql b/models/base/google/configuration/information_schema_organization_options_changes.sql
@@ -7,7 +7,7 @@
       ask your administrator to grant you the
 
 
-  BigQuery Admin  (roles/bigquery.admin) IAM role on organization.
+  BigQuery Admin  (roles/bigquery.admin) IAM role on your organization.
 
 
 

diff --git a/models/base/google/jobs/information_schema_jobs.sql b/models/base/google/jobs/information_schema_jobs.sql
@@ -20,20 +20,20 @@
     {% if project_list()|length > 0 -%}
         {% for project in project_list() -%}
 
-    SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info
+    SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info
     FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS`
 
         {% if not loop.last %}UNION ALL{% endif %}
         {% endfor %}
     {%- else %}
 
-    SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info
+    SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info
     FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS`
 
     {%- endif %}
     )
     SELECT
-    bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info,
+    bi_engine_statistics, cache_hit, creation_time, destination_table, dml_statistics, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info,
     FROM
     base
 
diff --git a/models/base/google/jobs/information_schema_jobs.yml b/models/base/google/jobs/information_schema_jobs.yml
@@ -31,7 +31,8 @@ models:
   - description: Details of any errors as ErrorProto objects.
     name: error_result
     type: RECORD
-  - description: The ID of the job. For example, bquxjob_1234.
+  - description: "The ID of the job if a job was created. Otherwise, the query ID\
+      \ of a query using short\n        query mode. For example, bquxjob_1234."
     name: job_id
     type: STRING
   - description: "Query\n          stages of the job.\n\n        Note: This column's\
@@ -76,8 +77,7 @@ models:
       \ the location of the reservation\nRESERVATION_NAME: the name of the reservation"
     name: reservation_id
     type: STRING
-  - description: "Details about the session\n        in which this job ran, if any.\
-      \ (Preview)"
+  - description: "Details about the session\n        in which this job ran, if any."
     name: session_info
     type: RECORD
   - description: "The start time of this job, in milliseconds since the epoch. This\
@@ -135,6 +135,17 @@ models:
       \ job. (Preview)"
     name: materialized_view_statistics
     type: RECORD
+  - description: "job_creation_reason.code : Specifies the high level reason why a\
+      \ job was created.\n        Possible values are:\n        \nREQUESTED: job creation\
+      \ was requested.\nLONG_RUNNING: the query request ran beyond a system defined\
+      \ timeout\n            specified by the\n            timeoutMs\n           \
+      \ field in the QueryRequest. As a result it was considered a long running\n\
+      \            operation for which a job was created.\nLARGE_RESULTS: the results\
+      \ from the query cannot fit in the in-line\n            response.\nOTHER: the\
+      \ system has determined that the query needs to be executed as a\n         \
+      \   job."
+    name: job_creation_reason
+    type: RECORD
   - description: "query_info.resource_warning : The warning message that appears if\
       \ the resource usage during query processing is above the internal threshold\
       \ of the system. A successful query job can have the resource_warning field\

diff --git a/models/base/google/jobs/information_schema_jobs_by_folder.sql b/models/base/google/jobs/information_schema_jobs_by_folder.sql
@@ -20,20 +20,20 @@
     {% if project_list()|length > 0 -%}
         {% for project in project_list() -%}
 
-    SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info
+    SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info
     FROM `{{ project | trim }}`.`region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_FOLDER`
 
         {% if not loop.last %}UNION ALL{% endif %}
         {% endfor %}
     {%- else %}
 
-    SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info
+    SELECT bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info
     FROM `region-{{ var('bq_region') }}`.`INFORMATION_SCHEMA`.`JOBS_BY_FOLDER`
 
     {%- endif %}
     )
     SELECT
-    bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, query_info,
+    bi_engine_statistics, cache_hit, creation_time, destination_table, end_time, error_result, job_id, job_stages, job_type, labels, parent_job_id, priority, project_id, project_number, query, referenced_tables, reservation_id, session_info, start_time, state, statement_type, timeline, total_bytes_billed, total_bytes_processed, total_modified_partitions, total_slot_ms, transaction_id, user_email, transferred_bytes, materialized_view_statistics, job_creation_reason, query_info,
     FROM
     base
 
diff --git a/models/base/google/jobs/information_schema_jobs_by_folder.yml b/models/base/google/jobs/information_schema_jobs_by_folder.yml
@@ -23,7 +23,8 @@ models:
   - description: Details of any errors as ErrorProto objects.
     name: error_result
     type: RECORD
-  - description: The ID of the job. For example, bquxjob_1234.
+  - description: "The ID of the job if a job was created. Otherwise, the query ID\
+      \ of a query using short\n        query mode. For example, bquxjob_1234."
     name: job_id
     type: STRING
   - description: "Query\n          stages of the job.\n\n        Note: This column's\
@@ -68,8 +69,7 @@ models:
       \ the location of the reservation\nRESERVATION_NAME: the name of the reservation"
     name: reservation_id
     type: STRING
-  - description: "Details about the session\n        in which this job ran, if any.\
-      \ (Preview)"
+  - description: "Details about the session\n        in which this job ran, if any."
     name: session_info
     type: RECORD
   - description: "The start time of this job, in milliseconds since the epoch. This\
@@ -127,6 +127,17 @@ models:
       \ job. (Preview)"
     name: materialized_view_statistics
     type: RECORD
+  - description: "job_creation_reason.code : Specifies the high level reason why a\
+      \ job was created.\n        Possible values are:\n        \nREQUESTED: job creation\
+      \ was requested.\nLONG_RUNNING: the query request ran beyond a system defined\
+      \ timeout\n            specified by the\n            timeoutMs\n           \
+      \ field in the QueryRequest. As a result it was considered a long running\n\
+      \            operation for which a job was created.\nLARGE_RESULTS: the results\
+      \ from the query cannot fit in the in-line\n            response.\nOTHER: the\
+      \ system has determined that the query needs to be executed as a\n         \
+      \   job."
+    name: job_creation_reason
+    type: RECORD
   - description: "query_info.resource_warning : The warning message that appears if\
       \ the resource usage during query processing is above the internal threshold\
       \ of the system. A successful query job can have the resource_warning field\
Original file line number	Diff line number	Diff line change
Expand Up		@@ -7,7 +7,7 @@
		ask your administrator to grant you the


		BigQuery Admin (roles/bigquery.admin) IAM role on organization.
		BigQuery Admin (roles/bigquery.admin) IAM role on your organization.



Expand Down