diff --git a/.changes/v0.6.0.md b/.changes/v0.6.0.md new file mode 100644 index 0000000..9a0afde --- /dev/null +++ b/.changes/v0.6.0.md @@ -0,0 +1,9 @@ +## dbt-bigquery-monitoring v0.6.0 - August 19, 2024 + +### Breaking Changes + +- Add an default output dataset and model with properties" ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0)) + +### Contributors +- [@Kayrnt](https://github.com/Kayrnt) ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0)) + diff --git a/CHANGELOG.md b/CHANGELOG.md index d23bdbb..b50d59f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-bigquery-monitoring v0.6.0 - August 19, 2024 + +### Breaking Changes + +- Add an default output dataset and model with properties" ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0)) + +### Contributors +- [@Kayrnt](https://github.com/Kayrnt) ([#0](https://github.com/bqbooster/dbt-bigquery-monitoring/issues/0)) + + ## dbt-bigquery-monitoring v0.5.3 - August 12, 2024 ### Docs diff --git a/README.md b/README.md index 16a5b0c..36a555f 100644 --- a/README.md +++ b/README.md @@ -93,16 +93,19 @@ Following settings are defined as `dbt_project_variable` (**Environment variable ##### Environment - `input_gcp_projects` (**DBT_BQ_MONITORING_GCP_PROJECTS**) : list of GCP projects to monitor (default: `[]`) - `bq_region` (**DBT_BQ_MONITORING_REGION**) : region where the monitored projects are located (default: `us`) +- `output_dataset` (**DBT_BQ_MONITORING_OUTPUT_DATASET**) : dataset where the models will be materialized (default: `dbt_bigquery_monitoring`) ##### Pricing - `use_flat_pricing` (**DBT_BQ_MONITORING_USE_FLAT_PRICING**) : whether to use flat pricing or not (default: `true`) -- `per_billed_tb_price` (**DBT_BQ_MONITORING_PER_BILLED_TB_PRICE**) : price per billed TB (default: `6,25`) -- `free_tb_per_month` (**DBT_BQ_MONITORING_FREE_TB_PER_MONTH**) : free TB per month (default: `1`) -- `hourly_slot_price` (**DBT_BQ_MONITORING_HOURLY_SLOT_PRICE**) : price per slot per hour (default: `0.04`) -- `active_logical_storage_gb_price` (**DBT_BQ_MONITORING_ACTIVE_LOGICAL_STORAGE_GB_PRICE**) : price per active logical storage GB (default: `0.02`) -- `long_term_logical_storage_gb_price` (**DBT_BQ_MONITORING_LONG_TERM_LOGICAL_STORAGE_GB_PRICE**) : price per long term logical storage GB (default: `0.01`) -- `active_physical_storage_gb_price` (**DBT_BQ_MONITORING_ACTIVE_PHYSICAL_STORAGE_GB_PRICE**) : price per active physical storage GB (default: `0.04`) -- `long_term_physical_storage_gb_price` (**DBT_BQ_MONITORING_LONG_TERM_PHYSICAL_STORAGE_GB_PRICE**) : price per long term physical storage GB (default: `0.02`) +- `per_billed_tb_price` (**DBT_BQ_MONITORING_PER_BILLED_TB_PRICE**) : price in US dollars per billed TB of data processed (default: `6,25`) +- `free_tb_per_month` (**DBT_BQ_MONITORING_FREE_TB_PER_MONTH**) : free on demand compute quota TB per month (default: `1`) +- `hourly_slot_price` (**DBT_BQ_MONITORING_HOURLY_SLOT_PRICE**) : hourly price in US dollars per slot per hour (default: `0.04`) +- `active_logical_storage_gb_price` (**DBT_BQ_MONITORING_ACTIVE_LOGICAL_STORAGE_GB_PRICE**) : monthly price in US dollars per active logical storage GB (default: `0.02`) +- `long_term_logical_storage_gb_price` (**DBT_BQ_MONITORING_LONG_TERM_LOGICAL_STORAGE_GB_PRICE**) : monthly price in US dollars per long term logical storage GB (default: `0.01`) +- `active_physical_storage_gb_price` (**DBT_BQ_MONITORING_ACTIVE_PHYSICAL_STORAGE_GB_PRICE**) : monthly price in US dollars per active physical storage GB (default: `0.04`) +- `long_term_physical_storage_gb_price` (**DBT_BQ_MONITORING_LONG_TERM_PHYSICAL_STORAGE_GB_PRICE**) : monthly price in US dollars per long term physical storage GB (default: `0.02`) +- `bi_engine_gb_hourly_price` (**DBT_BQ_MONITORING_BI_ENGINE_GB_HOURLY_PRICE**): hourly price in US dollars per BI engine GB of memory (default: `0.0416`) + - `free_storage_gb_per_month` (**DBT_BQ_MONITORING_FREE_STORAGE_GB_PER_MONTH**) : free storage GB per month (default: `10`) ###### Package diff --git a/dbt_project.yml b/dbt_project.yml index 785e825..5cd6d4f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -14,6 +14,7 @@ clean-targets: models: +start: Jan 1 2017 dbt_bigquery_monitoring: + +schema: "{{ env_var('DBT_BQ_MONITORING_OUTPUT_DATASET', 'dbt_bigquery_monitoring') }}" +tags: - "dbt-bigquery-monitoring" +on_schema_change: "append_new_columns" @@ -31,6 +32,7 @@ vars: # Environment configuration bq_region: "{{ env_var('DBT_BQ_MONITORING_REGION', 'us') }}" input_gcp_projects: "{{ env_var('DBT_BQ_MONITORING_GCP_PROJECTS') }}" # example: ['my-gcp-project', 'my-gcp-project2'] + output_dataset: "{{ env_var('DBT_BQ_MONITORING_OUTPUT_DATASET', 'dbt_bigquery_monitoring') }}" # https://cloud.google.com/bigquery/pricing#pricing # On-demand compute (analysis) pricing use_flat_pricing: "{{ env_var('DBT_BQ_MONITORING_USE_FLAT_PRICING', true) }}" @@ -45,6 +47,8 @@ vars: active_physical_storage_gb_price: "{{ env_var('DBT_BQ_MONITORING_ACTIVE_PHYSICAL_STORAGE_GB_PRICE', 0.04) }}" long_term_physical_storage_gb_price: "{{ env_var('DBT_BQ_MONITORING_LONG_TERM_PHYSICAL_STORAGE_GB_PRICE', 0.02) }}" free_storage_gb_per_month: "{{ env_var('DBT_BQ_MONITORING_FREE_STORAGE_GB_PER_MONTH', 10) }}" + # BI Engine pricing + bi_engine_gb_hourly_price: "{{ env_var('DBT_BQ_MONITORING_BI_ENGINE_GB_HOURLY_PRICE', 0.0416) }}" # Project input configuration lookback_window_days: "{{ env_var('DBT_BQ_MONITORING_LOOKBACK_WINDOW_DAYS', 7) }}" diff --git a/models/global/datamart/dbt_bigquery_monitoring_options.sql b/models/global/datamart/dbt_bigquery_monitoring_options.sql new file mode 100644 index 0000000..a62aaeb --- /dev/null +++ b/models/global/datamart/dbt_bigquery_monitoring_options.sql @@ -0,0 +1,32 @@ +{{ + config( + materialized='table', + ) +}} + +{%- set options = [ + 'bq_region', + 'input_gcp_projects', + 'output_dataset', + 'use_flat_pricing', + 'per_billed_tb_price', + 'free_tb_per_month', + 'hourly_slot_price', + 'prefer_physical_pricing_model', + 'active_logical_storage_gb_price', + 'long_term_logical_storage_gb_price', + 'active_physical_storage_gb_price', + 'long_term_physical_storage_gb_price', + 'free_storage_gb_per_month', + 'bi_engine_gb_hourly_price', + 'lookback_window_days', + 'output_materialization', + 'output_limit_size', +] %} + +{% for option in options %} +SELECT "{{ option }}" as option_label, "{{ var(option) }}" as option_value +{% if not loop.last %} +UNION ALL +{% endif %} +{% endfor %} diff --git a/models/global/datamart/dbt_bigquery_monitoring_options.yml b/models/global/datamart/dbt_bigquery_monitoring_options.yml new file mode 100644 index 0000000..f8fe351 --- /dev/null +++ b/models/global/datamart/dbt_bigquery_monitoring_options.yml @@ -0,0 +1,10 @@ +version: 2 + +models: + - name: bigquery_monitoring_options + description: This model contains current configuration options and values for dbt BigQuery monitoring extension. + columns: + - name: option_label + description: The label of the configuration option. + - name: option_value + description: The value of the configuration option.