From 00cb95c10e62fc867edfc3727fc0d50ff127ffa9 Mon Sep 17 00:00:00 2001 From: tlento Date: Fri, 13 Oct 2023 18:13:07 -0700 Subject: [PATCH 1/3] Update dbt adapter dependencies for 1.7.0rc1 SQL engine test environments do not currently build due to the cutover to dbt-core 1.7.0rc1. Now that the other adapter packages we depend on have a 1.7.0rc1 deployed, we can update our dependencies so that all of these will build and run. The only remaining holdout is duckdb, which we already have running on 1.6. --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f7d732ccd7..f3578d0709 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,19 +79,19 @@ dbt-postgres = [ ] dbt-bigquery = [ - "dbt-bigquery~=1.6.0", + "dbt-bigquery~=1.7.0rc1", ] dbt-databricks = [ - "dbt-databricks==1.6.0", + "dbt-databricks~=1.7.0rc1", ] dbt-redshift = [ - "dbt-redshift~=1.6.0", + "dbt-redshift~=1.7.0rc1", ] dbt-snowflake = [ - "dbt-snowflake~=1.6.0", + "dbt-snowflake~=1.7.0rc1", ] dbt-duckdb = [ From a7ae8bcca3fcbf8039c687303dc1c8e440fa8eee Mon Sep 17 00:00:00 2001 From: tlento Date: Fri, 13 Oct 2023 18:15:18 -0700 Subject: [PATCH 2/3] Update snapshots for distinct values plans for Databricks The distinct values plans could not be updated for Databricks on the final run due to an outage. These were auto-generated via the usual overwrite-snapshots mechanism. --- .../test_distinct_values__plan0.sql | 525 +++++------------- .../test_distinct_values__plan0_optimized.sql | 24 +- 2 files changed, 143 insertions(+), 406 deletions(-) diff --git a/metricflow/test/snapshots/test_dataflow_to_sql_plan.py/SqlQueryPlan/Databricks/test_distinct_values__plan0.sql b/metricflow/test/snapshots/test_dataflow_to_sql_plan.py/SqlQueryPlan/Databricks/test_distinct_values__plan0.sql index 86ba05d2c1..51c90f927c 100644 --- a/metricflow/test/snapshots/test_dataflow_to_sql_plan.py/SqlQueryPlan/Databricks/test_distinct_values__plan0.sql +++ b/metricflow/test/snapshots/test_dataflow_to_sql_plan.py/SqlQueryPlan/Databricks/test_distinct_values__plan0.sql @@ -1,404 +1,143 @@ -- Order By ['listing__country_latest'] Limit 100 SELECT - subq_10.listing__country_latest + subq_2.listing__country_latest FROM ( -- Pass Only Elements: -- ['listing__country_latest'] SELECT - subq_9.listing__country_latest + subq_1.listing__country_latest FROM ( - -- Compute Metrics via Expressions + -- Constrain Output with WHERE SELECT - subq_8.listing__country_latest - , subq_8.bookings + subq_0.ds__day + , subq_0.ds__week + , subq_0.ds__month + , subq_0.ds__quarter + , subq_0.ds__year + , subq_0.ds__extract_year + , subq_0.ds__extract_quarter + , subq_0.ds__extract_month + , subq_0.ds__extract_week + , subq_0.ds__extract_day + , subq_0.ds__extract_dow + , subq_0.ds__extract_doy + , subq_0.created_at__day + , subq_0.created_at__week + , subq_0.created_at__month + , subq_0.created_at__quarter + , subq_0.created_at__year + , subq_0.created_at__extract_year + , subq_0.created_at__extract_quarter + , subq_0.created_at__extract_month + , subq_0.created_at__extract_week + , subq_0.created_at__extract_day + , subq_0.created_at__extract_dow + , subq_0.created_at__extract_doy + , subq_0.listing__ds__day + , subq_0.listing__ds__week + , subq_0.listing__ds__month + , subq_0.listing__ds__quarter + , subq_0.listing__ds__year + , subq_0.listing__ds__extract_year + , subq_0.listing__ds__extract_quarter + , subq_0.listing__ds__extract_month + , subq_0.listing__ds__extract_week + , subq_0.listing__ds__extract_day + , subq_0.listing__ds__extract_dow + , subq_0.listing__ds__extract_doy + , subq_0.listing__created_at__day + , subq_0.listing__created_at__week + , subq_0.listing__created_at__month + , subq_0.listing__created_at__quarter + , subq_0.listing__created_at__year + , subq_0.listing__created_at__extract_year + , subq_0.listing__created_at__extract_quarter + , subq_0.listing__created_at__extract_month + , subq_0.listing__created_at__extract_week + , subq_0.listing__created_at__extract_day + , subq_0.listing__created_at__extract_dow + , subq_0.listing__created_at__extract_doy + , subq_0.listing + , subq_0.user + , subq_0.listing__user + , subq_0.country_latest + , subq_0.is_lux_latest + , subq_0.capacity_latest + , subq_0.listing__country_latest + , subq_0.listing__is_lux_latest + , subq_0.listing__capacity_latest + , subq_0.listings + , subq_0.largest_listing + , subq_0.smallest_listing FROM ( - -- Aggregate Measures + -- Read Elements From Semantic Model 'listings_latest' SELECT - subq_7.listing__country_latest - , SUM(subq_7.bookings) AS bookings - FROM ( - -- Pass Only Elements: - -- ['bookings', 'listing__country_latest'] - SELECT - subq_6.listing__country_latest - , subq_6.bookings - FROM ( - -- Join Standard Outputs - SELECT - subq_2.listing AS listing - , subq_5.country_latest AS listing__country_latest - , subq_2.bookings AS bookings - FROM ( - -- Pass Only Elements: - -- ['bookings', 'listing'] - SELECT - subq_1.listing - , subq_1.bookings - FROM ( - -- Metric Time Dimension 'ds' - SELECT - subq_0.ds__day - , subq_0.ds__week - , subq_0.ds__month - , subq_0.ds__quarter - , subq_0.ds__year - , subq_0.ds__extract_year - , subq_0.ds__extract_quarter - , subq_0.ds__extract_month - , subq_0.ds__extract_week - , subq_0.ds__extract_day - , subq_0.ds__extract_dow - , subq_0.ds__extract_doy - , subq_0.ds_partitioned__day - , subq_0.ds_partitioned__week - , subq_0.ds_partitioned__month - , subq_0.ds_partitioned__quarter - , subq_0.ds_partitioned__year - , subq_0.ds_partitioned__extract_year - , subq_0.ds_partitioned__extract_quarter - , subq_0.ds_partitioned__extract_month - , subq_0.ds_partitioned__extract_week - , subq_0.ds_partitioned__extract_day - , subq_0.ds_partitioned__extract_dow - , subq_0.ds_partitioned__extract_doy - , subq_0.paid_at__day - , subq_0.paid_at__week - , subq_0.paid_at__month - , subq_0.paid_at__quarter - , subq_0.paid_at__year - , subq_0.paid_at__extract_year - , subq_0.paid_at__extract_quarter - , subq_0.paid_at__extract_month - , subq_0.paid_at__extract_week - , subq_0.paid_at__extract_day - , subq_0.paid_at__extract_dow - , subq_0.paid_at__extract_doy - , subq_0.booking__ds__day - , subq_0.booking__ds__week - , subq_0.booking__ds__month - , subq_0.booking__ds__quarter - , subq_0.booking__ds__year - , subq_0.booking__ds__extract_year - , subq_0.booking__ds__extract_quarter - , subq_0.booking__ds__extract_month - , subq_0.booking__ds__extract_week - , subq_0.booking__ds__extract_day - , subq_0.booking__ds__extract_dow - , subq_0.booking__ds__extract_doy - , subq_0.booking__ds_partitioned__day - , subq_0.booking__ds_partitioned__week - , subq_0.booking__ds_partitioned__month - , subq_0.booking__ds_partitioned__quarter - , subq_0.booking__ds_partitioned__year - , subq_0.booking__ds_partitioned__extract_year - , subq_0.booking__ds_partitioned__extract_quarter - , subq_0.booking__ds_partitioned__extract_month - , subq_0.booking__ds_partitioned__extract_week - , subq_0.booking__ds_partitioned__extract_day - , subq_0.booking__ds_partitioned__extract_dow - , subq_0.booking__ds_partitioned__extract_doy - , subq_0.booking__paid_at__day - , subq_0.booking__paid_at__week - , subq_0.booking__paid_at__month - , subq_0.booking__paid_at__quarter - , subq_0.booking__paid_at__year - , subq_0.booking__paid_at__extract_year - , subq_0.booking__paid_at__extract_quarter - , subq_0.booking__paid_at__extract_month - , subq_0.booking__paid_at__extract_week - , subq_0.booking__paid_at__extract_day - , subq_0.booking__paid_at__extract_dow - , subq_0.booking__paid_at__extract_doy - , subq_0.ds__day AS metric_time__day - , subq_0.ds__week AS metric_time__week - , subq_0.ds__month AS metric_time__month - , subq_0.ds__quarter AS metric_time__quarter - , subq_0.ds__year AS metric_time__year - , subq_0.ds__extract_year AS metric_time__extract_year - , subq_0.ds__extract_quarter AS metric_time__extract_quarter - , subq_0.ds__extract_month AS metric_time__extract_month - , subq_0.ds__extract_week AS metric_time__extract_week - , subq_0.ds__extract_day AS metric_time__extract_day - , subq_0.ds__extract_dow AS metric_time__extract_dow - , subq_0.ds__extract_doy AS metric_time__extract_doy - , subq_0.listing - , subq_0.guest - , subq_0.host - , subq_0.booking__listing - , subq_0.booking__guest - , subq_0.booking__host - , subq_0.is_instant - , subq_0.booking__is_instant - , subq_0.bookings - , subq_0.instant_bookings - , subq_0.booking_value - , subq_0.max_booking_value - , subq_0.min_booking_value - , subq_0.bookers - , subq_0.average_booking_value - , subq_0.referred_bookings - , subq_0.median_booking_value - , subq_0.booking_value_p99 - , subq_0.discrete_booking_value_p99 - , subq_0.approximate_continuous_booking_value_p99 - , subq_0.approximate_discrete_booking_value_p99 - FROM ( - -- Read Elements From Semantic Model 'bookings_source' - SELECT - 1 AS bookings - , CASE WHEN is_instant THEN 1 ELSE 0 END AS instant_bookings - , bookings_source_src_10001.booking_value - , bookings_source_src_10001.booking_value AS max_booking_value - , bookings_source_src_10001.booking_value AS min_booking_value - , bookings_source_src_10001.guest_id AS bookers - , bookings_source_src_10001.booking_value AS average_booking_value - , bookings_source_src_10001.booking_value AS booking_payments - , CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings - , bookings_source_src_10001.booking_value AS median_booking_value - , bookings_source_src_10001.booking_value AS booking_value_p99 - , bookings_source_src_10001.booking_value AS discrete_booking_value_p99 - , bookings_source_src_10001.booking_value AS approximate_continuous_booking_value_p99 - , bookings_source_src_10001.booking_value AS approximate_discrete_booking_value_p99 - , bookings_source_src_10001.is_instant - , DATE_TRUNC('day', bookings_source_src_10001.ds) AS ds__day - , DATE_TRUNC('week', bookings_source_src_10001.ds) AS ds__week - , DATE_TRUNC('month', bookings_source_src_10001.ds) AS ds__month - , DATE_TRUNC('quarter', bookings_source_src_10001.ds) AS ds__quarter - , DATE_TRUNC('year', bookings_source_src_10001.ds) AS ds__year - , EXTRACT(year FROM bookings_source_src_10001.ds) AS ds__extract_year - , EXTRACT(quarter FROM bookings_source_src_10001.ds) AS ds__extract_quarter - , EXTRACT(month FROM bookings_source_src_10001.ds) AS ds__extract_month - , EXTRACT(week FROM bookings_source_src_10001.ds) AS ds__extract_week - , EXTRACT(day FROM bookings_source_src_10001.ds) AS ds__extract_day - , EXTRACT(dow FROM bookings_source_src_10001.ds) AS ds__extract_dow - , EXTRACT(doy FROM bookings_source_src_10001.ds) AS ds__extract_doy - , DATE_TRUNC('day', bookings_source_src_10001.ds_partitioned) AS ds_partitioned__day - , DATE_TRUNC('week', bookings_source_src_10001.ds_partitioned) AS ds_partitioned__week - , DATE_TRUNC('month', bookings_source_src_10001.ds_partitioned) AS ds_partitioned__month - , DATE_TRUNC('quarter', bookings_source_src_10001.ds_partitioned) AS ds_partitioned__quarter - , DATE_TRUNC('year', bookings_source_src_10001.ds_partitioned) AS ds_partitioned__year - , EXTRACT(year FROM bookings_source_src_10001.ds_partitioned) AS ds_partitioned__extract_year - , EXTRACT(quarter FROM bookings_source_src_10001.ds_partitioned) AS ds_partitioned__extract_quarter - , EXTRACT(month FROM bookings_source_src_10001.ds_partitioned) AS ds_partitioned__extract_month - , EXTRACT(week FROM bookings_source_src_10001.ds_partitioned) AS ds_partitioned__extract_week - , EXTRACT(day FROM bookings_source_src_10001.ds_partitioned) AS ds_partitioned__extract_day - , EXTRACT(dow FROM bookings_source_src_10001.ds_partitioned) AS ds_partitioned__extract_dow - , EXTRACT(doy FROM bookings_source_src_10001.ds_partitioned) AS ds_partitioned__extract_doy - , DATE_TRUNC('day', bookings_source_src_10001.paid_at) AS paid_at__day - , DATE_TRUNC('week', bookings_source_src_10001.paid_at) AS paid_at__week - , DATE_TRUNC('month', bookings_source_src_10001.paid_at) AS paid_at__month - , DATE_TRUNC('quarter', bookings_source_src_10001.paid_at) AS paid_at__quarter - , DATE_TRUNC('year', bookings_source_src_10001.paid_at) AS paid_at__year - , EXTRACT(year FROM bookings_source_src_10001.paid_at) AS paid_at__extract_year - , EXTRACT(quarter FROM bookings_source_src_10001.paid_at) AS paid_at__extract_quarter - , EXTRACT(month FROM bookings_source_src_10001.paid_at) AS paid_at__extract_month - , EXTRACT(week FROM bookings_source_src_10001.paid_at) AS paid_at__extract_week - , EXTRACT(day FROM bookings_source_src_10001.paid_at) AS paid_at__extract_day - , EXTRACT(dow FROM bookings_source_src_10001.paid_at) AS paid_at__extract_dow - , EXTRACT(doy FROM bookings_source_src_10001.paid_at) AS paid_at__extract_doy - , bookings_source_src_10001.is_instant AS booking__is_instant - , DATE_TRUNC('day', bookings_source_src_10001.ds) AS booking__ds__day - , DATE_TRUNC('week', bookings_source_src_10001.ds) AS booking__ds__week - , DATE_TRUNC('month', bookings_source_src_10001.ds) AS booking__ds__month - , DATE_TRUNC('quarter', bookings_source_src_10001.ds) AS booking__ds__quarter - , DATE_TRUNC('year', bookings_source_src_10001.ds) AS booking__ds__year - , EXTRACT(year FROM bookings_source_src_10001.ds) AS booking__ds__extract_year - , EXTRACT(quarter FROM bookings_source_src_10001.ds) AS booking__ds__extract_quarter - , EXTRACT(month FROM bookings_source_src_10001.ds) AS booking__ds__extract_month - , EXTRACT(week FROM bookings_source_src_10001.ds) AS booking__ds__extract_week - , EXTRACT(day FROM bookings_source_src_10001.ds) AS booking__ds__extract_day - , EXTRACT(dow FROM bookings_source_src_10001.ds) AS booking__ds__extract_dow - , EXTRACT(doy FROM bookings_source_src_10001.ds) AS booking__ds__extract_doy - , DATE_TRUNC('day', bookings_source_src_10001.ds_partitioned) AS booking__ds_partitioned__day - , DATE_TRUNC('week', bookings_source_src_10001.ds_partitioned) AS booking__ds_partitioned__week - , DATE_TRUNC('month', bookings_source_src_10001.ds_partitioned) AS booking__ds_partitioned__month - , DATE_TRUNC('quarter', bookings_source_src_10001.ds_partitioned) AS booking__ds_partitioned__quarter - , DATE_TRUNC('year', bookings_source_src_10001.ds_partitioned) AS booking__ds_partitioned__year - , EXTRACT(year FROM bookings_source_src_10001.ds_partitioned) AS booking__ds_partitioned__extract_year - , EXTRACT(quarter FROM bookings_source_src_10001.ds_partitioned) AS booking__ds_partitioned__extract_quarter - , EXTRACT(month FROM bookings_source_src_10001.ds_partitioned) AS booking__ds_partitioned__extract_month - , EXTRACT(week FROM bookings_source_src_10001.ds_partitioned) AS booking__ds_partitioned__extract_week - , EXTRACT(day FROM bookings_source_src_10001.ds_partitioned) AS booking__ds_partitioned__extract_day - , EXTRACT(dow FROM bookings_source_src_10001.ds_partitioned) AS booking__ds_partitioned__extract_dow - , EXTRACT(doy FROM bookings_source_src_10001.ds_partitioned) AS booking__ds_partitioned__extract_doy - , DATE_TRUNC('day', bookings_source_src_10001.paid_at) AS booking__paid_at__day - , DATE_TRUNC('week', bookings_source_src_10001.paid_at) AS booking__paid_at__week - , DATE_TRUNC('month', bookings_source_src_10001.paid_at) AS booking__paid_at__month - , DATE_TRUNC('quarter', bookings_source_src_10001.paid_at) AS booking__paid_at__quarter - , DATE_TRUNC('year', bookings_source_src_10001.paid_at) AS booking__paid_at__year - , EXTRACT(year FROM bookings_source_src_10001.paid_at) AS booking__paid_at__extract_year - , EXTRACT(quarter FROM bookings_source_src_10001.paid_at) AS booking__paid_at__extract_quarter - , EXTRACT(month FROM bookings_source_src_10001.paid_at) AS booking__paid_at__extract_month - , EXTRACT(week FROM bookings_source_src_10001.paid_at) AS booking__paid_at__extract_week - , EXTRACT(day FROM bookings_source_src_10001.paid_at) AS booking__paid_at__extract_day - , EXTRACT(dow FROM bookings_source_src_10001.paid_at) AS booking__paid_at__extract_dow - , EXTRACT(doy FROM bookings_source_src_10001.paid_at) AS booking__paid_at__extract_doy - , bookings_source_src_10001.listing_id AS listing - , bookings_source_src_10001.guest_id AS guest - , bookings_source_src_10001.host_id AS host - , bookings_source_src_10001.listing_id AS booking__listing - , bookings_source_src_10001.guest_id AS booking__guest - , bookings_source_src_10001.host_id AS booking__host - FROM ***************************.fct_bookings bookings_source_src_10001 - ) subq_0 - ) subq_1 - ) subq_2 - LEFT OUTER JOIN ( - -- Pass Only Elements: - -- ['country_latest', 'listing'] - SELECT - subq_4.listing - , subq_4.country_latest - FROM ( - -- Metric Time Dimension 'ds' - SELECT - subq_3.ds__day - , subq_3.ds__week - , subq_3.ds__month - , subq_3.ds__quarter - , subq_3.ds__year - , subq_3.ds__extract_year - , subq_3.ds__extract_quarter - , subq_3.ds__extract_month - , subq_3.ds__extract_week - , subq_3.ds__extract_day - , subq_3.ds__extract_dow - , subq_3.ds__extract_doy - , subq_3.created_at__day - , subq_3.created_at__week - , subq_3.created_at__month - , subq_3.created_at__quarter - , subq_3.created_at__year - , subq_3.created_at__extract_year - , subq_3.created_at__extract_quarter - , subq_3.created_at__extract_month - , subq_3.created_at__extract_week - , subq_3.created_at__extract_day - , subq_3.created_at__extract_dow - , subq_3.created_at__extract_doy - , subq_3.listing__ds__day - , subq_3.listing__ds__week - , subq_3.listing__ds__month - , subq_3.listing__ds__quarter - , subq_3.listing__ds__year - , subq_3.listing__ds__extract_year - , subq_3.listing__ds__extract_quarter - , subq_3.listing__ds__extract_month - , subq_3.listing__ds__extract_week - , subq_3.listing__ds__extract_day - , subq_3.listing__ds__extract_dow - , subq_3.listing__ds__extract_doy - , subq_3.listing__created_at__day - , subq_3.listing__created_at__week - , subq_3.listing__created_at__month - , subq_3.listing__created_at__quarter - , subq_3.listing__created_at__year - , subq_3.listing__created_at__extract_year - , subq_3.listing__created_at__extract_quarter - , subq_3.listing__created_at__extract_month - , subq_3.listing__created_at__extract_week - , subq_3.listing__created_at__extract_day - , subq_3.listing__created_at__extract_dow - , subq_3.listing__created_at__extract_doy - , subq_3.ds__day AS metric_time__day - , subq_3.ds__week AS metric_time__week - , subq_3.ds__month AS metric_time__month - , subq_3.ds__quarter AS metric_time__quarter - , subq_3.ds__year AS metric_time__year - , subq_3.ds__extract_year AS metric_time__extract_year - , subq_3.ds__extract_quarter AS metric_time__extract_quarter - , subq_3.ds__extract_month AS metric_time__extract_month - , subq_3.ds__extract_week AS metric_time__extract_week - , subq_3.ds__extract_day AS metric_time__extract_day - , subq_3.ds__extract_dow AS metric_time__extract_dow - , subq_3.ds__extract_doy AS metric_time__extract_doy - , subq_3.listing - , subq_3.user - , subq_3.listing__user - , subq_3.country_latest - , subq_3.is_lux_latest - , subq_3.capacity_latest - , subq_3.listing__country_latest - , subq_3.listing__is_lux_latest - , subq_3.listing__capacity_latest - , subq_3.listings - , subq_3.largest_listing - , subq_3.smallest_listing - FROM ( - -- Read Elements From Semantic Model 'listings_latest' - SELECT - 1 AS listings - , listings_latest_src_10004.capacity AS largest_listing - , listings_latest_src_10004.capacity AS smallest_listing - , DATE_TRUNC('day', listings_latest_src_10004.created_at) AS ds__day - , DATE_TRUNC('week', listings_latest_src_10004.created_at) AS ds__week - , DATE_TRUNC('month', listings_latest_src_10004.created_at) AS ds__month - , DATE_TRUNC('quarter', listings_latest_src_10004.created_at) AS ds__quarter - , DATE_TRUNC('year', listings_latest_src_10004.created_at) AS ds__year - , EXTRACT(year FROM listings_latest_src_10004.created_at) AS ds__extract_year - , EXTRACT(quarter FROM listings_latest_src_10004.created_at) AS ds__extract_quarter - , EXTRACT(month FROM listings_latest_src_10004.created_at) AS ds__extract_month - , EXTRACT(week FROM listings_latest_src_10004.created_at) AS ds__extract_week - , EXTRACT(day FROM listings_latest_src_10004.created_at) AS ds__extract_day - , EXTRACT(dow FROM listings_latest_src_10004.created_at) AS ds__extract_dow - , EXTRACT(doy FROM listings_latest_src_10004.created_at) AS ds__extract_doy - , DATE_TRUNC('day', listings_latest_src_10004.created_at) AS created_at__day - , DATE_TRUNC('week', listings_latest_src_10004.created_at) AS created_at__week - , DATE_TRUNC('month', listings_latest_src_10004.created_at) AS created_at__month - , DATE_TRUNC('quarter', listings_latest_src_10004.created_at) AS created_at__quarter - , DATE_TRUNC('year', listings_latest_src_10004.created_at) AS created_at__year - , EXTRACT(year FROM listings_latest_src_10004.created_at) AS created_at__extract_year - , EXTRACT(quarter FROM listings_latest_src_10004.created_at) AS created_at__extract_quarter - , EXTRACT(month FROM listings_latest_src_10004.created_at) AS created_at__extract_month - , EXTRACT(week FROM listings_latest_src_10004.created_at) AS created_at__extract_week - , EXTRACT(day FROM listings_latest_src_10004.created_at) AS created_at__extract_day - , EXTRACT(dow FROM listings_latest_src_10004.created_at) AS created_at__extract_dow - , EXTRACT(doy FROM listings_latest_src_10004.created_at) AS created_at__extract_doy - , listings_latest_src_10004.country AS country_latest - , listings_latest_src_10004.is_lux AS is_lux_latest - , listings_latest_src_10004.capacity AS capacity_latest - , DATE_TRUNC('day', listings_latest_src_10004.created_at) AS listing__ds__day - , DATE_TRUNC('week', listings_latest_src_10004.created_at) AS listing__ds__week - , DATE_TRUNC('month', listings_latest_src_10004.created_at) AS listing__ds__month - , DATE_TRUNC('quarter', listings_latest_src_10004.created_at) AS listing__ds__quarter - , DATE_TRUNC('year', listings_latest_src_10004.created_at) AS listing__ds__year - , EXTRACT(year FROM listings_latest_src_10004.created_at) AS listing__ds__extract_year - , EXTRACT(quarter FROM listings_latest_src_10004.created_at) AS listing__ds__extract_quarter - , EXTRACT(month FROM listings_latest_src_10004.created_at) AS listing__ds__extract_month - , EXTRACT(week FROM listings_latest_src_10004.created_at) AS listing__ds__extract_week - , EXTRACT(day FROM listings_latest_src_10004.created_at) AS listing__ds__extract_day - , EXTRACT(dow FROM listings_latest_src_10004.created_at) AS listing__ds__extract_dow - , EXTRACT(doy FROM listings_latest_src_10004.created_at) AS listing__ds__extract_doy - , DATE_TRUNC('day', listings_latest_src_10004.created_at) AS listing__created_at__day - , DATE_TRUNC('week', listings_latest_src_10004.created_at) AS listing__created_at__week - , DATE_TRUNC('month', listings_latest_src_10004.created_at) AS listing__created_at__month - , DATE_TRUNC('quarter', listings_latest_src_10004.created_at) AS listing__created_at__quarter - , DATE_TRUNC('year', listings_latest_src_10004.created_at) AS listing__created_at__year - , EXTRACT(year FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_year - , EXTRACT(quarter FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_quarter - , EXTRACT(month FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_month - , EXTRACT(week FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_week - , EXTRACT(day FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_day - , EXTRACT(dow FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_dow - , EXTRACT(doy FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_doy - , listings_latest_src_10004.country AS listing__country_latest - , listings_latest_src_10004.is_lux AS listing__is_lux_latest - , listings_latest_src_10004.capacity AS listing__capacity_latest - , listings_latest_src_10004.listing_id AS listing - , listings_latest_src_10004.user_id AS user - , listings_latest_src_10004.user_id AS listing__user - FROM ***************************.dim_listings_latest listings_latest_src_10004 - ) subq_3 - ) subq_4 - ) subq_5 - ON - subq_2.listing = subq_5.listing - ) subq_6 - ) subq_7 - GROUP BY - subq_7.listing__country_latest - ) subq_8 - ) subq_9 -) subq_10 -ORDER BY subq_10.listing__country_latest + 1 AS listings + , listings_latest_src_10004.capacity AS largest_listing + , listings_latest_src_10004.capacity AS smallest_listing + , DATE_TRUNC('day', listings_latest_src_10004.created_at) AS ds__day + , DATE_TRUNC('week', listings_latest_src_10004.created_at) AS ds__week + , DATE_TRUNC('month', listings_latest_src_10004.created_at) AS ds__month + , DATE_TRUNC('quarter', listings_latest_src_10004.created_at) AS ds__quarter + , DATE_TRUNC('year', listings_latest_src_10004.created_at) AS ds__year + , EXTRACT(year FROM listings_latest_src_10004.created_at) AS ds__extract_year + , EXTRACT(quarter FROM listings_latest_src_10004.created_at) AS ds__extract_quarter + , EXTRACT(month FROM listings_latest_src_10004.created_at) AS ds__extract_month + , EXTRACT(week FROM listings_latest_src_10004.created_at) AS ds__extract_week + , EXTRACT(day FROM listings_latest_src_10004.created_at) AS ds__extract_day + , EXTRACT(dow FROM listings_latest_src_10004.created_at) AS ds__extract_dow + , EXTRACT(doy FROM listings_latest_src_10004.created_at) AS ds__extract_doy + , DATE_TRUNC('day', listings_latest_src_10004.created_at) AS created_at__day + , DATE_TRUNC('week', listings_latest_src_10004.created_at) AS created_at__week + , DATE_TRUNC('month', listings_latest_src_10004.created_at) AS created_at__month + , DATE_TRUNC('quarter', listings_latest_src_10004.created_at) AS created_at__quarter + , DATE_TRUNC('year', listings_latest_src_10004.created_at) AS created_at__year + , EXTRACT(year FROM listings_latest_src_10004.created_at) AS created_at__extract_year + , EXTRACT(quarter FROM listings_latest_src_10004.created_at) AS created_at__extract_quarter + , EXTRACT(month FROM listings_latest_src_10004.created_at) AS created_at__extract_month + , EXTRACT(week FROM listings_latest_src_10004.created_at) AS created_at__extract_week + , EXTRACT(day FROM listings_latest_src_10004.created_at) AS created_at__extract_day + , EXTRACT(dow FROM listings_latest_src_10004.created_at) AS created_at__extract_dow + , EXTRACT(doy FROM listings_latest_src_10004.created_at) AS created_at__extract_doy + , listings_latest_src_10004.country AS country_latest + , listings_latest_src_10004.is_lux AS is_lux_latest + , listings_latest_src_10004.capacity AS capacity_latest + , DATE_TRUNC('day', listings_latest_src_10004.created_at) AS listing__ds__day + , DATE_TRUNC('week', listings_latest_src_10004.created_at) AS listing__ds__week + , DATE_TRUNC('month', listings_latest_src_10004.created_at) AS listing__ds__month + , DATE_TRUNC('quarter', listings_latest_src_10004.created_at) AS listing__ds__quarter + , DATE_TRUNC('year', listings_latest_src_10004.created_at) AS listing__ds__year + , EXTRACT(year FROM listings_latest_src_10004.created_at) AS listing__ds__extract_year + , EXTRACT(quarter FROM listings_latest_src_10004.created_at) AS listing__ds__extract_quarter + , EXTRACT(month FROM listings_latest_src_10004.created_at) AS listing__ds__extract_month + , EXTRACT(week FROM listings_latest_src_10004.created_at) AS listing__ds__extract_week + , EXTRACT(day FROM listings_latest_src_10004.created_at) AS listing__ds__extract_day + , EXTRACT(dow FROM listings_latest_src_10004.created_at) AS listing__ds__extract_dow + , EXTRACT(doy FROM listings_latest_src_10004.created_at) AS listing__ds__extract_doy + , DATE_TRUNC('day', listings_latest_src_10004.created_at) AS listing__created_at__day + , DATE_TRUNC('week', listings_latest_src_10004.created_at) AS listing__created_at__week + , DATE_TRUNC('month', listings_latest_src_10004.created_at) AS listing__created_at__month + , DATE_TRUNC('quarter', listings_latest_src_10004.created_at) AS listing__created_at__quarter + , DATE_TRUNC('year', listings_latest_src_10004.created_at) AS listing__created_at__year + , EXTRACT(year FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_year + , EXTRACT(quarter FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_quarter + , EXTRACT(month FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_month + , EXTRACT(week FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_week + , EXTRACT(day FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_day + , EXTRACT(dow FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_dow + , EXTRACT(doy FROM listings_latest_src_10004.created_at) AS listing__created_at__extract_doy + , listings_latest_src_10004.country AS listing__country_latest + , listings_latest_src_10004.is_lux AS listing__is_lux_latest + , listings_latest_src_10004.capacity AS listing__capacity_latest + , listings_latest_src_10004.listing_id AS listing + , listings_latest_src_10004.user_id AS user + , listings_latest_src_10004.user_id AS listing__user + FROM ***************************.dim_listings_latest listings_latest_src_10004 + ) subq_0 + WHERE listing__country_latest = 'us' + ) subq_1 + GROUP BY + subq_1.listing__country_latest +) subq_2 +ORDER BY subq_2.listing__country_latest DESC LIMIT 100 diff --git a/metricflow/test/snapshots/test_dataflow_to_sql_plan.py/SqlQueryPlan/Databricks/test_distinct_values__plan0_optimized.sql b/metricflow/test/snapshots/test_dataflow_to_sql_plan.py/SqlQueryPlan/Databricks/test_distinct_values__plan0_optimized.sql index 328d6a2d79..ec0d94dac2 100644 --- a/metricflow/test/snapshots/test_dataflow_to_sql_plan.py/SqlQueryPlan/Databricks/test_distinct_values__plan0_optimized.sql +++ b/metricflow/test/snapshots/test_dataflow_to_sql_plan.py/SqlQueryPlan/Databricks/test_distinct_values__plan0_optimized.sql @@ -1,19 +1,17 @@ --- Join Standard Outputs --- Pass Only Elements: --- ['bookings', 'listing__country_latest'] --- Aggregate Measures --- Compute Metrics via Expressions +-- Constrain Output with WHERE -- Pass Only Elements: -- ['listing__country_latest'] -- Order By ['listing__country_latest'] Limit 100 SELECT - listings_latest_src_10004.country AS listing__country_latest -FROM ***************************.fct_bookings bookings_source_src_10001 -LEFT OUTER JOIN - ***************************.dim_listings_latest listings_latest_src_10004 -ON - bookings_source_src_10001.listing_id = listings_latest_src_10004.listing_id + listing__country_latest +FROM ( + -- Read Elements From Semantic Model 'listings_latest' + SELECT + country AS listing__country_latest + FROM ***************************.dim_listings_latest listings_latest_src_10004 +) subq_3 +WHERE listing__country_latest = 'us' GROUP BY - listings_latest_src_10004.country -ORDER BY listing__country_latest + listing__country_latest +ORDER BY listing__country_latest DESC LIMIT 100 From 85abc5a20a648eaf560f54b3502cbd9c5d8da70a Mon Sep 17 00:00:00 2001 From: tlento Date: Fri, 13 Oct 2023 18:44:53 -0700 Subject: [PATCH 3/3] Consolidate Databricks testing to a single action For historical reasons we were testing both cluster connection types and sql warehouse connection types. Since cutting our connection management over to dbt adapters and removing all DDL from MetricFlow we no longer need to test in both runtimes, so this removes the cluster connection tests and brings our Databricks environment variable in line with the other engines. --- ...ine-populate-persistent-source-schema.yaml | 2 +- .github/workflows/cd-sql-engine-tests.yaml | 25 +++---------------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/.github/workflows/cd-sql-engine-populate-persistent-source-schema.yaml b/.github/workflows/cd-sql-engine-populate-persistent-source-schema.yaml index e62b12b16d..b8a779cca4 100644 --- a/.github/workflows/cd-sql-engine-populate-persistent-source-schema.yaml +++ b/.github/workflows/cd-sql-engine-populate-persistent-source-schema.yaml @@ -93,7 +93,7 @@ jobs: uses: ./.github/actions/run-mf-tests with: python-version: ${{ env.PYTHON_VERSION }} - mf_sql_engine_url: ${{ secrets.MF_DATABRICKS_SQL_WAREHOUSE_URL }} + mf_sql_engine_url: ${{ secrets.MF_DATABRICKS_URL }} mf_sql_engine_password: ${{ secrets.MF_DATABRICKS_PWD }} parallelism: 1 make-target: "populate-persistent-source-schema-databricks" diff --git a/.github/workflows/cd-sql-engine-tests.yaml b/.github/workflows/cd-sql-engine-tests.yaml index 40c17e10e0..cca613c629 100644 --- a/.github/workflows/cd-sql-engine-tests.yaml +++ b/.github/workflows/cd-sql-engine-tests.yaml @@ -72,28 +72,9 @@ jobs: additional-pytest-options: ${{ env.ADDITIONAL_PYTEST_OPTIONS }} make-target: "test-bigquery" - databricks-cluster-tests: + databricks-tests: environment: DW_INTEGRATION_TESTS - name: Databricks Cluster Tests - if: ${{ github.event.action != 'labeled' || github.event.label.name == 'run_mf_sql_engine_tests' }} - runs-on: ubuntu-latest - steps: - - name: Check-out the repo - uses: actions/checkout@v3 - - - name: Test w/ Python Python ${{ env.PYTHON_VERSION }} - uses: ./.github/actions/run-mf-tests - with: - python-version: ${{ env.PYTHON_VERSION }} - mf_sql_engine_url: ${{ secrets.MF_DATABRICKS_CLUSTER_URL }} - mf_sql_engine_password: ${{ secrets.MF_DATABRICKS_PWD }} - parallelism: ${{ env.EXTERNAL_ENGINE_TEST_PARALLELISM }} - additional-pytest-options: ${{ env.ADDITIONAL_PYTEST_OPTIONS }} - make-target: "test-databricks" - - databricks-sql-warehouse-tests: - environment: DW_INTEGRATION_TESTS - name: Databricks SQL Warehouse Tests + name: Databricks Tests if: ${{ github.event.action != 'labeled' || github.event.label.name == 'run_mf_sql_engine_tests' }} runs-on: ubuntu-latest steps: @@ -104,7 +85,7 @@ jobs: uses: ./.github/actions/run-mf-tests with: python-version: ${{ env.PYTHON_VERSION }} - mf_sql_engine_url: ${{ secrets.MF_DATABRICKS_SQL_WAREHOUSE_URL }} + mf_sql_engine_url: ${{ secrets.MF_DATABRICKS_URL }} mf_sql_engine_password: ${{ secrets.MF_DATABRICKS_PWD }} parallelism: ${{ env.EXTERNAL_ENGINE_TEST_PARALLELISM }} additional-pytest-options: ${{ env.ADDITIONAL_PYTEST_OPTIONS }}