From 3675431ba6517e4913a31186db5fec0b70221e15 Mon Sep 17 00:00:00 2001 From: Mike Gouline <1960272+gouline@users.noreply.github.com> Date: Wed, 10 Jan 2024 13:40:01 +1100 Subject: [PATCH] Fixes and enhancements to sandbox and tests (#207) --- .gitignore | 2 + Makefile | 6 +- requirements-test.txt | 2 + sandbox/.gitignore | 1 - sandbox/Dockerfile | 12 +- sandbox/docker-compose.yml | 7 +- sandbox/models/schema.yml | 1 - sandbox/requirements.txt | 3 - tests/fixtures/exposure/.gitignore | 1 - .../a_look_at_your_customers_table.yml | 370 ++++++++++++++++++ .../exposure/collection/our_analytics.yml | 49 +++ .../exposures.yml} | 0 tests/fixtures/exposure/type/card/1.yml | 26 ++ tests/fixtures/exposure/type/card/10.yml | 25 ++ tests/fixtures/exposure/type/card/11.yml | 25 ++ tests/fixtures/exposure/type/card/12.yml | 25 ++ tests/fixtures/exposure/type/card/13.yml | 25 ++ tests/fixtures/exposure/type/card/14.yml | 25 ++ tests/fixtures/exposure/type/card/15.yml | 25 ++ tests/fixtures/exposure/type/card/16.yml | 25 ++ tests/fixtures/exposure/type/card/17.yml | 25 ++ tests/fixtures/exposure/type/card/2.yml | 25 ++ tests/fixtures/exposure/type/card/3.yml | 25 ++ tests/fixtures/exposure/type/card/4.yml | 25 ++ tests/fixtures/exposure/type/card/5.yml | 25 ++ tests/fixtures/exposure/type/card/6.yml | 25 ++ tests/fixtures/exposure/type/card/7.yml | 25 ++ tests/fixtures/exposure/type/card/8.yml | 25 ++ tests/fixtures/exposure/type/card/9.yml | 25 ++ tests/fixtures/exposure/type/dashboard/1.yml | 25 ++ tests/test_metabase.py | 74 +++- 31 files changed, 953 insertions(+), 26 deletions(-) delete mode 100644 sandbox/.gitignore delete mode 100644 sandbox/requirements.txt delete mode 100644 tests/fixtures/exposure/.gitignore create mode 100644 tests/fixtures/exposure/collection/a_look_at_your_customers_table.yml create mode 100644 tests/fixtures/exposure/collection/our_analytics.yml rename tests/fixtures/exposure/{baseline_test_exposures.yml => default/exposures.yml} (100%) create mode 100644 tests/fixtures/exposure/type/card/1.yml create mode 100644 tests/fixtures/exposure/type/card/10.yml create mode 100644 tests/fixtures/exposure/type/card/11.yml create mode 100644 tests/fixtures/exposure/type/card/12.yml create mode 100644 tests/fixtures/exposure/type/card/13.yml create mode 100644 tests/fixtures/exposure/type/card/14.yml create mode 100644 tests/fixtures/exposure/type/card/15.yml create mode 100644 tests/fixtures/exposure/type/card/16.yml create mode 100644 tests/fixtures/exposure/type/card/17.yml create mode 100644 tests/fixtures/exposure/type/card/2.yml create mode 100644 tests/fixtures/exposure/type/card/3.yml create mode 100644 tests/fixtures/exposure/type/card/4.yml create mode 100644 tests/fixtures/exposure/type/card/5.yml create mode 100644 tests/fixtures/exposure/type/card/6.yml create mode 100644 tests/fixtures/exposure/type/card/7.yml create mode 100644 tests/fixtures/exposure/type/card/8.yml create mode 100644 tests/fixtures/exposure/type/card/9.yml create mode 100644 tests/fixtures/exposure/type/dashboard/1.yml diff --git a/.gitignore b/.gitignore index 603bf9bc..366131fa 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ share/python-wheels/ *.egg MANIFEST */_version.py +tmp/ # PyInstaller # Usually these files are written by a python script from a template @@ -133,4 +134,5 @@ dmypy.json # dbt dbt_modules/ logs/ +target/ .user.yml diff --git a/Makefile b/Makefile index 733d2a7e..c9e6771f 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,7 @@ clean: requirements: python3 -m pip install \ -r requirements.txt \ - -r requirements-test.txt \ - -r sandbox/requirements.txt + -r requirements-test.txt .PHONY: requirements fix-fmt: @@ -47,6 +46,9 @@ test: python3 -m unittest tests .PHONY: test +pre: fix check test +.PHONY: pre + dist-check: build twine check dist/* .PHONY: dist-check diff --git a/requirements-test.txt b/requirements-test.txt index 078ee83c..c35ccd81 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -5,5 +5,7 @@ black>=23.11.0 isort>=5.12.0 pylint>=3.0.2 mypy>=1.7.1 +molot>=1.0.0,<1.1.0 +dbt-postgres>=1.7.4,<2.0.0 types-requests types-PyYAML diff --git a/sandbox/.gitignore b/sandbox/.gitignore deleted file mode 100644 index b83d2226..00000000 --- a/sandbox/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target/ diff --git a/sandbox/Dockerfile b/sandbox/Dockerfile index e198bb9e..3ebf646a 100644 --- a/sandbox/Dockerfile +++ b/sandbox/Dockerfile @@ -2,15 +2,11 @@ FROM python:3.11-slim-bullseye WORKDIR /app -COPY requirements.txt ./ +COPY --from=root requirements.txt ./ +COPY --from=root requirements-test.txt ./ -RUN python3 -m pip install -r requirements.txt +RUN pip install -r requirements.txt -r requirements-test.txt -COPY entrypoint.py ./ -COPY profiles.yml ./ - -COPY models/ ./models/ -COPY seeds/ ./seeds/ -COPY dbt_project.yml ./ +WORKDIR /app/sandbox ENTRYPOINT ["python3", "entrypoint.py"] diff --git a/sandbox/docker-compose.yml b/sandbox/docker-compose.yml index 57067959..a7063e7d 100644 --- a/sandbox/docker-compose.yml +++ b/sandbox/docker-compose.yml @@ -41,7 +41,10 @@ services: condition: service_healthy app: - build: . + build: + context: . + additional_contexts: + root: .. command: init environment: - POSTGRES_USER=${POSTGRES_USER:-} @@ -58,7 +61,7 @@ services: networks: - common volumes: - - ./target:/app/target + - .:/app/sandbox depends_on: postgres: condition: service_healthy diff --git a/sandbox/models/schema.yml b/sandbox/models/schema.yml index 57486fe4..59184256 100644 --- a/sandbox/models/schema.yml +++ b/sandbox/models/schema.yml @@ -59,7 +59,6 @@ models: description: Date (UTC) that the order was placed - name: status - description: '{{ doc("orders_status") }}' tests: - accepted_values: values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] diff --git a/sandbox/requirements.txt b/sandbox/requirements.txt deleted file mode 100644 index 1c76a539..00000000 --- a/sandbox/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -molot>=1.0.0,<1.1.0 -requests>=2.26.0,<3.0.0 -dbt-postgres>=1.7.4,<2.0.0 diff --git a/tests/fixtures/exposure/.gitignore b/tests/fixtures/exposure/.gitignore deleted file mode 100644 index b6104ec0..00000000 --- a/tests/fixtures/exposure/.gitignore +++ /dev/null @@ -1 +0,0 @@ -exposures.yml diff --git a/tests/fixtures/exposure/collection/a_look_at_your_customers_table.yml b/tests/fixtures/exposure/collection/a_look_at_your_customers_table.yml new file mode 100644 index 00000000..917e8df8 --- /dev/null +++ b/tests/fixtures/exposure/collection/a_look_at_your_customers_table.yml @@ -0,0 +1,370 @@ +version: 2 +exposures: + - name: a_look_at_your_customers_table + label: A look at your customers table + description: '### Dashboard Cards: 18 + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __1__ + + + Created On: __2021-07-21T08:01:37.328519Z__' + type: dashboard + url: http://localhost:3000/dashboard/1 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: customer_lifetime_value_over_time + label: Customer_lifetime_value over time + description: '### Visualization: Line + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __11__ + + + Created On: __2021-07-21T08:01:38.52841Z__' + type: analysis + url: http://localhost:3000/card/11 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: customer_lifetime_value_over_time_1 + label: Customer_lifetime_value over time + description: '### Visualization: Line + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __9__ + + + Created On: __2021-07-21T08:01:38.189846Z__' + type: analysis + url: http://localhost:3000/card/9 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: customers_added_in_the_last_30_days + label: Customers added in the last 30 days + description: '### Visualization: Scalar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __4__ + + + Created On: __2021-07-21T08:01:37.535187Z__' + type: analysis + url: http://localhost:3000/card/4 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: customers_added_in_the_last_30_days_1 + label: Customers added in the last 30 days + description: '### Visualization: Scalar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __5__ + + + Created On: __2021-07-21T08:01:37.630969Z__' + type: analysis + url: http://localhost:3000/card/5 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: customers_by_customer_lifetime_value + label: Customers by customer_lifetime_value + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __7__ + + + Created On: __2021-07-21T08:01:37.858387Z__' + type: analysis + url: http://localhost:3000/card/7 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: customers_by_number_of_orders + label: Customers by number_of_orders + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __6__ + + + Created On: __2021-07-21T08:01:37.754606Z__' + type: analysis + url: http://localhost:3000/card/6 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: number_of_orders_over_time + label: Number_of_orders over time + description: '### Visualization: Line + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __8__ + + + Created On: __2021-07-21T08:01:38.016244Z__' + type: analysis + url: http://localhost:3000/card/8 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: number_of_orders_over_time_1 + label: Number_of_orders over time + description: '### Visualization: Line + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __10__ + + + Created On: __2021-07-21T08:01:38.347535Z__' + type: analysis + url: http://localhost:3000/card/10 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: timestamp_by_day_of_the_week + label: Timestamp by day of the week + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __12__ + + + Created On: __2021-07-21T08:01:38.662485Z__' + type: analysis + url: http://localhost:3000/card/12 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: timestamp_by_day_of_the_week_1 + label: Timestamp by day of the week + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __13__ + + + Created On: __2021-07-21T08:01:38.808186Z__' + type: analysis + url: http://localhost:3000/card/13 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: timestamp_by_month_of_the_year + label: Timestamp by month of the year + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __15__ + + + Created On: __2021-07-21T08:01:39.084258Z__' + type: analysis + url: http://localhost:3000/card/15 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: timestamp_by_month_of_the_year_1 + label: Timestamp by month of the year + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __14__ + + + Created On: __2021-07-21T08:01:38.938688Z__' + type: analysis + url: http://localhost:3000/card/14 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: timestamp_by_quarter_of_the_year + label: Timestamp by quarter of the year + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __17__ + + + Created On: __2021-07-21T08:01:39.371067Z__' + type: analysis + url: http://localhost:3000/card/17 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: timestamp_by_quarter_of_the_year_1 + label: Timestamp by quarter of the year + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __16__ + + + Created On: __2021-07-21T08:01:39.228965Z__' + type: analysis + url: http://localhost:3000/card/16 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: total_customers + label: Total customers + description: '### Visualization: Scalar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __3__ + + + Created On: __2021-07-21T08:01:37.434243Z__' + type: analysis + url: http://localhost:3000/card/3 + maturity: medium + owner: + name: '' + email: '' + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/collection/our_analytics.yml b/tests/fixtures/exposure/collection/our_analytics.yml new file mode 100644 index 00000000..87808be0 --- /dev/null +++ b/tests/fixtures/exposure/collection/our_analytics.yml @@ -0,0 +1,49 @@ +version: 2 +exposures: + - name: customers__sum_of_number_of_orders_and_average_of_number_of_orders__grouped_by_most_recent_order__day_ + label: Customers, Sum of Number Of Orders and Average of Number Of Orders, Grouped + by Most Recent Order (day) + description: '### Visualization: Line + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __1__ + + + Created On: __2021-07-21T08:00:47.453351Z__' + type: analysis + url: http://localhost:3000/card/1 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') + - name: orders__count + label: Orders, Count + description: '### Visualization: Scalar + + + Nice KPI + + + #### Metadata + + + Metabase Id: __2__ + + + Created On: __2021-07-21T08:01:14.290572Z__' + type: analysis + url: http://localhost:3000/card/2 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('orders') diff --git a/tests/fixtures/exposure/baseline_test_exposures.yml b/tests/fixtures/exposure/default/exposures.yml similarity index 100% rename from tests/fixtures/exposure/baseline_test_exposures.yml rename to tests/fixtures/exposure/default/exposures.yml diff --git a/tests/fixtures/exposure/type/card/1.yml b/tests/fixtures/exposure/type/card/1.yml new file mode 100644 index 00000000..2da55424 --- /dev/null +++ b/tests/fixtures/exposure/type/card/1.yml @@ -0,0 +1,26 @@ +version: 2 +exposures: + - name: customers__sum_of_number_of_orders_and_average_of_number_of_orders__grouped_by_most_recent_order__day_ + label: Customers, Sum of Number Of Orders and Average of Number Of Orders, Grouped + by Most Recent Order (day) + description: '### Visualization: Line + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __1__ + + + Created On: __2021-07-21T08:00:47.453351Z__' + type: analysis + url: http://localhost:3000/card/1 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/10.yml b/tests/fixtures/exposure/type/card/10.yml new file mode 100644 index 00000000..2d3e8094 --- /dev/null +++ b/tests/fixtures/exposure/type/card/10.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: number_of_orders_over_time_1 + label: Number_of_orders over time + description: '### Visualization: Line + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __10__ + + + Created On: __2021-07-21T08:01:38.347535Z__' + type: analysis + url: http://localhost:3000/card/10 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/11.yml b/tests/fixtures/exposure/type/card/11.yml new file mode 100644 index 00000000..1758e0d0 --- /dev/null +++ b/tests/fixtures/exposure/type/card/11.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: customer_lifetime_value_over_time + label: Customer_lifetime_value over time + description: '### Visualization: Line + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __11__ + + + Created On: __2021-07-21T08:01:38.52841Z__' + type: analysis + url: http://localhost:3000/card/11 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/12.yml b/tests/fixtures/exposure/type/card/12.yml new file mode 100644 index 00000000..1563b540 --- /dev/null +++ b/tests/fixtures/exposure/type/card/12.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: timestamp_by_day_of_the_week + label: Timestamp by day of the week + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __12__ + + + Created On: __2021-07-21T08:01:38.662485Z__' + type: analysis + url: http://localhost:3000/card/12 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/13.yml b/tests/fixtures/exposure/type/card/13.yml new file mode 100644 index 00000000..6f4bfe78 --- /dev/null +++ b/tests/fixtures/exposure/type/card/13.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: timestamp_by_day_of_the_week_1 + label: Timestamp by day of the week + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __13__ + + + Created On: __2021-07-21T08:01:38.808186Z__' + type: analysis + url: http://localhost:3000/card/13 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/14.yml b/tests/fixtures/exposure/type/card/14.yml new file mode 100644 index 00000000..057a00e0 --- /dev/null +++ b/tests/fixtures/exposure/type/card/14.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: timestamp_by_month_of_the_year_1 + label: Timestamp by month of the year + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __14__ + + + Created On: __2021-07-21T08:01:38.938688Z__' + type: analysis + url: http://localhost:3000/card/14 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/15.yml b/tests/fixtures/exposure/type/card/15.yml new file mode 100644 index 00000000..041d2190 --- /dev/null +++ b/tests/fixtures/exposure/type/card/15.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: timestamp_by_month_of_the_year + label: Timestamp by month of the year + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __15__ + + + Created On: __2021-07-21T08:01:39.084258Z__' + type: analysis + url: http://localhost:3000/card/15 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/16.yml b/tests/fixtures/exposure/type/card/16.yml new file mode 100644 index 00000000..c24bea5e --- /dev/null +++ b/tests/fixtures/exposure/type/card/16.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: timestamp_by_quarter_of_the_year_1 + label: Timestamp by quarter of the year + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __16__ + + + Created On: __2021-07-21T08:01:39.228965Z__' + type: analysis + url: http://localhost:3000/card/16 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/17.yml b/tests/fixtures/exposure/type/card/17.yml new file mode 100644 index 00000000..1738544d --- /dev/null +++ b/tests/fixtures/exposure/type/card/17.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: timestamp_by_quarter_of_the_year + label: Timestamp by quarter of the year + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __17__ + + + Created On: __2021-07-21T08:01:39.371067Z__' + type: analysis + url: http://localhost:3000/card/17 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/2.yml b/tests/fixtures/exposure/type/card/2.yml new file mode 100644 index 00000000..0073b9e7 --- /dev/null +++ b/tests/fixtures/exposure/type/card/2.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: orders__count + label: Orders, Count + description: '### Visualization: Scalar + + + Nice KPI + + + #### Metadata + + + Metabase Id: __2__ + + + Created On: __2021-07-21T08:01:14.290572Z__' + type: analysis + url: http://localhost:3000/card/2 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('orders') diff --git a/tests/fixtures/exposure/type/card/3.yml b/tests/fixtures/exposure/type/card/3.yml new file mode 100644 index 00000000..d25b6419 --- /dev/null +++ b/tests/fixtures/exposure/type/card/3.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: total_customers + label: Total customers + description: '### Visualization: Scalar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __3__ + + + Created On: __2021-07-21T08:01:37.434243Z__' + type: analysis + url: http://localhost:3000/card/3 + maturity: medium + owner: + name: '' + email: '' + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/4.yml b/tests/fixtures/exposure/type/card/4.yml new file mode 100644 index 00000000..5308f7a7 --- /dev/null +++ b/tests/fixtures/exposure/type/card/4.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: customers_added_in_the_last_30_days + label: Customers added in the last 30 days + description: '### Visualization: Scalar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __4__ + + + Created On: __2021-07-21T08:01:37.535187Z__' + type: analysis + url: http://localhost:3000/card/4 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/5.yml b/tests/fixtures/exposure/type/card/5.yml new file mode 100644 index 00000000..8816014b --- /dev/null +++ b/tests/fixtures/exposure/type/card/5.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: customers_added_in_the_last_30_days_1 + label: Customers added in the last 30 days + description: '### Visualization: Scalar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __5__ + + + Created On: __2021-07-21T08:01:37.630969Z__' + type: analysis + url: http://localhost:3000/card/5 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/6.yml b/tests/fixtures/exposure/type/card/6.yml new file mode 100644 index 00000000..4b1ff81f --- /dev/null +++ b/tests/fixtures/exposure/type/card/6.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: customers_by_number_of_orders + label: Customers by number_of_orders + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __6__ + + + Created On: __2021-07-21T08:01:37.754606Z__' + type: analysis + url: http://localhost:3000/card/6 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/7.yml b/tests/fixtures/exposure/type/card/7.yml new file mode 100644 index 00000000..f14f41e5 --- /dev/null +++ b/tests/fixtures/exposure/type/card/7.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: customers_by_customer_lifetime_value + label: Customers by customer_lifetime_value + description: '### Visualization: Bar + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __7__ + + + Created On: __2021-07-21T08:01:37.858387Z__' + type: analysis + url: http://localhost:3000/card/7 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/8.yml b/tests/fixtures/exposure/type/card/8.yml new file mode 100644 index 00000000..466178da --- /dev/null +++ b/tests/fixtures/exposure/type/card/8.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: number_of_orders_over_time + label: Number_of_orders over time + description: '### Visualization: Line + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __8__ + + + Created On: __2021-07-21T08:01:38.016244Z__' + type: analysis + url: http://localhost:3000/card/8 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/card/9.yml b/tests/fixtures/exposure/type/card/9.yml new file mode 100644 index 00000000..4195aaf8 --- /dev/null +++ b/tests/fixtures/exposure/type/card/9.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: customer_lifetime_value_over_time_1 + label: Customer_lifetime_value over time + description: '### Visualization: Line + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __9__ + + + Created On: __2021-07-21T08:01:38.189846Z__' + type: analysis + url: http://localhost:3000/card/9 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/fixtures/exposure/type/dashboard/1.yml b/tests/fixtures/exposure/type/dashboard/1.yml new file mode 100644 index 00000000..c77f39ee --- /dev/null +++ b/tests/fixtures/exposure/type/dashboard/1.yml @@ -0,0 +1,25 @@ +version: 2 +exposures: + - name: a_look_at_your_customers_table + label: A look at your customers table + description: '### Dashboard Cards: 18 + + + No description provided in Metabase + + + #### Metadata + + + Metabase Id: __1__ + + + Created On: __2021-07-21T08:01:37.328519Z__' + type: dashboard + url: http://localhost:3000/dashboard/1 + maturity: medium + owner: + name: dbtmetabase + email: user@example.com + depends_on: + - ref('customers') diff --git a/tests/test_metabase.py b/tests/test_metabase.py index 43669b58..45157955 100644 --- a/tests/test_metabase.py +++ b/tests/test_metabase.py @@ -12,6 +12,7 @@ from dbtmetabase.metabase import MetabaseClient, _ExportModelsJob, _ExtractExposuresJob FIXTURES_PATH = Path("tests") / "fixtures" +TMP_PATH = Path("tests") / "tmp" MODELS = [ MetabaseModel( @@ -266,9 +267,22 @@ def setUp(self): ) logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) + TMP_PATH.mkdir(exist_ok=True) + + def _assert_exposures(self, expected_path: Path, actual_path: Path): + with open(expected_path, encoding="utf-8") as f: + expected = yaml.safe_load(f) + with open(actual_path, encoding="utf-8") as f: + actual = yaml.safe_load(f) + + self.assertEqual( + sorted(expected["exposures"], key=itemgetter("name")), + actual["exposures"], + ) def test_exposures(self): - output_path = FIXTURES_PATH / "exposure" + fixtures_path = FIXTURES_PATH / "exposure" / "default" + output_path = TMP_PATH / "exposure" / "default" job = _ExtractExposuresJob( client=self.client, models=MODELS, @@ -280,15 +294,59 @@ def test_exposures(self): ) job.execute() - with open(output_path / "baseline_test_exposures.yml", encoding="utf-8") as f: - expected = yaml.safe_load(f) - with open(output_path / "exposures.yml", encoding="utf-8") as f: - actual = yaml.safe_load(f) + self._assert_exposures( + fixtures_path / "exposures.yml", + output_path / "exposures.yml", + ) - expected_exposures = sorted(expected["exposures"], key=itemgetter("name")) - actual_exposures = sorted(actual["exposures"], key=itemgetter("name")) + def test_exposures_collection_grouping(self): + fixtures_path = FIXTURES_PATH / "exposure" / "collection" + output_path = TMP_PATH / "exposure" / "collection" + job = _ExtractExposuresJob( + client=self.client, + models=MODELS, + output_path=str(output_path), + output_grouping="collection", + include_personal_collections=False, + collection_includes=None, + collection_excludes=None, + ) + job.execute() - self.assertEqual(expected_exposures, actual_exposures) + self._assert_exposures( + fixtures_path / "a_look_at_your_customers_table.yml", + output_path / "a_look_at_your_customers_table.yml", + ) + self._assert_exposures( + fixtures_path / "our_analytics.yml", + output_path / "our_analytics.yml", + ) + + def test_exposures_type_grouping(self): + fixtures_path = FIXTURES_PATH / "exposure" / "type" + output_path = TMP_PATH / "exposure" / "type" + job = _ExtractExposuresJob( + client=self.client, + models=MODELS, + output_path=str(output_path), + output_grouping="type", + include_personal_collections=False, + collection_includes=None, + collection_excludes=None, + ) + job.execute() + + for i in range(1, 18): + self._assert_exposures( + fixtures_path / "card" / f"{i}.yml", + output_path / "card" / f"{i}.yml", + ) + + for i in range(1, 2): + self._assert_exposures( + fixtures_path / "dashboard" / f"{i}.yml", + output_path / "dashboard" / f"{i}.yml", + ) def test_build_lookups(self): job = _ExportModelsJob(