Move sample_project into sandbox and anonymize test data (#205)

.gitignore

@@ -133,3 +133,4 @@ dmypy.json
 # dbt
 dbt_modules/
 logs/
+.user.yml

Makefile

@@ -9,7 +9,8 @@ clean:
 requirements:
 	python3 -m pip install \
 		-r requirements.txt \
-		-r requirements-test.txt
+		-r requirements-test.txt \
+		-r sandbox/requirements.txt
 .PHONY: requirements
 
 fix-fmt:
@@ -60,7 +61,11 @@ dev-install: build
 .PHONY: dev-install
 
 dev-sandbox-up:
-	( cd sandbox && docker-compose up --build --detach --wait )
+	( cd sandbox && docker-compose up --build --attach app )
+.PHONY: dev-sandbox-up
+
+dev-sandbox-down:
+	( cd sandbox && docker-compose down )
 .PHONY: dev-sandbox-up
 
 dev-sandbox-models:
@@ -74,27 +79,26 @@ dev-sandbox-models:
 .PHONY: dev-sandbox-models
 
 dev-sandbox-exposures:
-	rm -rf tests/fixtures/sample_project/models/exposures
-	mkdir -p tests/fixtures/sample_project/models/exposures
+	rm -rf sandbox/models/exposures
+	mkdir -p sandbox/models/exposures
 	( source sandbox/.env && python3 -m dbtmetabase exposures \
 		--dbt-manifest-path sandbox/target/manifest.json \
 		--dbt-database $$POSTGRES_DB \
 		--metabase-url http://localhost:$$MB_PORT \
 		--metabase-username $$MB_USER \
 		--metabase-password $$MB_PASSWORD \
-		--output-path tests/fixtures/sample_project/models/exposures \
+		--output-path sandbox/models/exposures \
 		--output-grouping collection )
 
-	( source sandbox/.env && cd tests/fixtures/sample_project && \
+	( source sandbox/.env && cd sandbox && \
 		POSTGRES_HOST=localhost \
 		POSTGRES_PORT=$$POSTGRES_PORT \
 		POSTGRES_USER=$$POSTGRES_USER \
 		POSTGRES_PASSWORD=$$POSTGRES_PASSWORD \
 		POSTGRES_DB=$$POSTGRES_DB \
 		POSTGRES_SCHEMA=$$POSTGRES_SCHEMA \
-		dbt docs generate --profiles-dir ../../../sandbox )
+		dbt docs generate )
 .PHONY: dev-sandbox-exposures
 
-dev-sandbox-down:
-	( cd sandbox && docker-compose down )
-.PHONY: dev-sandbox-up
+dev-sandbox-e2e: dev-sandbox-up dev-sandbox-models dev-sandbox-exposures dev-sandbox-down
+.PHONY: dev-sandbox-e2e

sandbox/Dockerfile

@@ -9,9 +9,8 @@ RUN python3 -m pip install -r requirements.txt
 COPY entrypoint.py ./
 COPY profiles.yml ./
 
-COPY --from=sample_project etc/ ./etc/
-COPY --from=sample_project models/ ./models/
-COPY --from=sample_project seeds/ ./seeds/
-COPY --from=sample_project dbt_project.yml ./
+COPY models/ ./models/
+COPY seeds/ ./seeds/
+COPY dbt_project.yml ./
 
 ENTRYPOINT ["python3", "entrypoint.py"]

sandbox/README.md

@@ -0,0 +1,7 @@
+# Sandbox
+
+Docker Compose setup for end-to-end testing with dbt, Metabase and PostgreSQL as target database.
+
+## Sample dbt project
+
+Started from the official [jaffle_shop](https://github.com/dbt-labs/jaffle_shop) sample with several modifications to test specific dbt-metabase features.

tests/fixtures/sample_project/dbt_project.yml → sandbox/dbt_project.yml

@@ -1,4 +1,4 @@
-name: "jaffle_shop"
+name: "sandbox"
 profile: "default"
 
 config-version: 2
@@ -13,7 +13,7 @@ clean-targets:
   - "logs"
 
 models:
-  jaffle_shop:
+  sandbox:
     materialized: table
     staging:
       materialized: view

sandbox/docker-compose.yml

@@ -41,10 +41,7 @@ services:
         condition: service_healthy
 
   app:
-    build:
-      context: .
-      additional_contexts:
-        sample_project: ../tests/fixtures/sample_project
+    build: .
     command: init
     environment:
       - POSTGRES_USER=${POSTGRES_USER:-}

sandbox/models/.gitignore

@@ -0,0 +1 @@
+/exposures/

setup.py

@@ -27,7 +27,7 @@ def requires_from_file(filename: str) -> list:
     entry_points={
         "console_scripts": ["dbt-metabase = dbtmetabase.__main__:cli"],
     },
-    packages=find_packages(exclude=["tests"]),
+    packages=find_packages(exclude=["tests", "sandbox"]),
     test_suite="tests",
     install_requires=requires_from_file("requirements.txt"),
     tests_require=requires_from_file("requirements-test.txt"),

tests/fixtures/exposure/baseline_test_exposures.yml

@@ -20,7 +20,7 @@ exposures:
   url: http://localhost:3000/card/1
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -43,7 +43,7 @@ exposures:
   url: http://localhost:3000/card/2
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('orders')
@@ -66,7 +66,7 @@ exposures:
   url: http://localhost:3000/dashboard/1
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -89,7 +89,7 @@ exposures:
   url: http://localhost:3000/card/11
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -112,7 +112,7 @@ exposures:
   url: http://localhost:3000/card/9
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -135,7 +135,7 @@ exposures:
   url: http://localhost:3000/card/4
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -158,7 +158,7 @@ exposures:
   url: http://localhost:3000/card/5
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -181,7 +181,7 @@ exposures:
   url: http://localhost:3000/card/7
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -204,7 +204,7 @@ exposures:
   url: http://localhost:3000/card/6
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -227,7 +227,7 @@ exposures:
   url: http://localhost:3000/card/8
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -250,7 +250,7 @@ exposures:
   url: http://localhost:3000/card/10
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -273,7 +273,7 @@ exposures:
   url: http://localhost:3000/card/12
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -296,7 +296,7 @@ exposures:
   url: http://localhost:3000/card/13
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -319,7 +319,7 @@ exposures:
   url: http://localhost:3000/card/15
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -342,7 +342,7 @@ exposures:
   url: http://localhost:3000/card/14
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -365,7 +365,7 @@ exposures:
   url: http://localhost:3000/card/17
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')
@@ -388,7 +388,7 @@ exposures:
   url: http://localhost:3000/card/16
   maturity: medium
   owner:
-    name: Alexander Butler
+    name: dbtmetabase
     email: [email protected]
   depends_on:
   - ref('customers')

tests/fixtures/mock_api/api/card/1.json

@@ -1 +1 @@
-{"description": null, "archived": false, "collection_position": null, "table_id": 7, "result_metadata": [{"semantic_type": null, "coercion_strategy": null, "unit": "day", "name": "most_recent_order", "field_ref": ["field", 42, {"temporal-unit": "day"}], "effective_type": "type/Date", "id": 42, "display_name": "most_recent_order", "fingerprint": {"global": {"distinct-count": 53, "nil%": 0.38}, "type": {"type/DateTime": {"earliest": "2018-01-09", "latest": "2018-04-09"}}}, "base_type": "type/Date"}, {"name": "sum", "display_name": "Sum of number_of_orders", "base_type": "type/Decimal", "semantic_type": "type/Quantity", "field_ref": ["aggregation", 0], "fingerprint": {"global": {"distinct-count": 7, "nil%": 0.01886792452830189}, "type": {"type/Number": {"min": 1.0, "q1": 1.0200644129064447, "q3": 2.3725029928215275, "max": 7.0, "sd": 1.25650644505132, "avg": 1.9038461538461537}}}}, {"name": "avg", "display_name": "Average of number_of_orders", "base_type": "type/Decimal", "semantic_type": "type/Quantity", "field_ref": ["aggregation", 1], "fingerprint": {"global": {"distinct-count": 8, "nil%": 0.01886792452830189}, "type": {"type/Number": {"min": 1.0, "q1": 1.0, "q3": 2.019031639883796, "max": 5.0, "sd": 0.7594097162561813, "avg": 1.573717948717949}}}}], "creator": {"email": "[email protected]", "first_name": "Alexander", "last_login": "2021-07-21T19:25:28.6083Z", "is_qbnewb": false, "is_superuser": true, "id": 1, "last_name": "Butler", "date_joined": "2021-07-21T05:38:53.637091Z", "common_name": "Alexander Butler"}, "can_write": true, "database_id": 2, "enable_embedding": false, "collection_id": null, "query_type": "query", "name": "Customers, Sum of Number Of Orders and Average of Number Of Orders, Grouped by Most Recent Order (day)", "dashboard_count": 0, "creator_id": 1, "updated_at": "2021-07-21T19:19:58.848286Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"type": "query", "database": 2, "query": {"source-table": 7, "breakout": [["field", 42, {"temporal-unit": "day"}]], "aggregation": [["sum", ["field", 40, null]], ["avg", ["field", 40, null]]]}}, "id": 1, "display": "line", "last-edit-info": {"id": 1, "email": "[email protected]", "first_name": "Alexander", "last_name": "Butler", "timestamp": "2021-07-21T08:00:47.4892Z"}, "visualization_settings": {"graph.series_labels": ["sum", "average"], "graph.metrics": ["sum", "avg"], "graph.dimensions": ["most_recent_order"], "graph.colors": ["#7172AD", "#EF8C8C"]}, "collection": null, "created_at": "2021-07-21T08:00:47.453351Z", "public_uuid": null}
+{"description": null, "archived": false, "collection_position": null, "table_id": 7, "result_metadata": [{"semantic_type": null, "coercion_strategy": null, "unit": "day", "name": "most_recent_order", "field_ref": ["field", 42, {"temporal-unit": "day"}], "effective_type": "type/Date", "id": 42, "display_name": "most_recent_order", "fingerprint": {"global": {"distinct-count": 53, "nil%": 0.38}, "type": {"type/DateTime": {"earliest": "2018-01-09", "latest": "2018-04-09"}}}, "base_type": "type/Date"}, {"name": "sum", "display_name": "Sum of number_of_orders", "base_type": "type/Decimal", "semantic_type": "type/Quantity", "field_ref": ["aggregation", 0], "fingerprint": {"global": {"distinct-count": 7, "nil%": 0.01886792452830189}, "type": {"type/Number": {"min": 1.0, "q1": 1.0200644129064447, "q3": 2.3725029928215275, "max": 7.0, "sd": 1.25650644505132, "avg": 1.9038461538461537}}}}, {"name": "avg", "display_name": "Average of number_of_orders", "base_type": "type/Decimal", "semantic_type": "type/Quantity", "field_ref": ["aggregation", 1], "fingerprint": {"global": {"distinct-count": 8, "nil%": 0.01886792452830189}, "type": {"type/Number": {"min": 1.0, "q1": 1.0, "q3": 2.019031639883796, "max": 5.0, "sd": 0.7594097162561813, "avg": 1.573717948717949}}}}], "creator": {"email": "[email protected]", "first_name": "dbtmetabase", "last_login": "2021-07-21T19:25:28.6083Z", "is_qbnewb": false, "is_superuser": true, "id": 1, "last_name": "", "date_joined": "2021-07-21T05:38:53.637091Z", "common_name": "dbtmetabase"}, "can_write": true, "database_id": 2, "enable_embedding": false, "collection_id": null, "query_type": "query", "name": "Customers, Sum of Number Of Orders and Average of Number Of Orders, Grouped by Most Recent Order (day)", "dashboard_count": 0, "creator_id": 1, "updated_at": "2021-07-21T19:19:58.848286Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"type": "query", "database": 2, "query": {"source-table": 7, "breakout": [["field", 42, {"temporal-unit": "day"}]], "aggregation": [["sum", ["field", 40, null]], ["avg", ["field", 40, null]]]}}, "id": 1, "display": "line", "last-edit-info": {"id": 1, "email": "[email protected]", "first_name": "dbtmetabase", "last_name": "", "timestamp": "2021-07-21T08:00:47.4892Z"}, "visualization_settings": {"graph.series_labels": ["sum", "average"], "graph.metrics": ["sum", "avg"], "graph.dimensions": ["most_recent_order"], "graph.colors": ["#7172AD", "#EF8C8C"]}, "collection": null, "created_at": "2021-07-21T08:00:47.453351Z", "public_uuid": null}

tests/fixtures/mock_api/api/card/10.json

@@ -1 +1 @@
-{"description": null, "archived": false, "collection_position": null, "table_id": 7, "result_metadata": [{"semantic_type": null, "coercion_strategy": null, "unit": "day", "name": "first_order", "field_ref": ["field", 39, {"temporal-unit": "day"}], "effective_type": "type/Date", "id": 39, "display_name": "first_order", "fingerprint": {"global": {"distinct-count": 47, "nil%": 0.38}, "type": {"type/DateTime": {"earliest": "2018-01-01", "latest": "2018-04-07"}}}, "base_type": "type/Date"}, {"name": "sum", "display_name": "Sum of number_of_orders", "base_type": "type/Decimal", "semantic_type": "type/Quantity", "field_ref": ["aggregation", 0], "fingerprint": {"global": {"distinct-count": 6, "nil%": 0.02127659574468085}, "type": {"type/Number": {"min": 1.0, "q1": 1.1073874870150486, "q3": 3.0, "max": 5.0, "sd": 1.2643380578934107, "avg": 2.152173913043478}}}}, {"name": "avg", "display_name": "Average of number_of_orders", "base_type": "type/Decimal", "semantic_type": "type/Quantity", "field_ref": ["aggregation", 1], "fingerprint": {"global": {"distinct-count": 8, "nil%": 0.02127659574468085}, "type": {"type/Number": {"min": 1.0, "q1": 1.019470273980893, "q3": 1.989426104361334, "max": 5.0, "sd": 0.7515349295958083, "avg": 1.596014492753623}}}}], "creator": {"email": "[email protected]", "first_name": "Alexander", "last_login": "2021-07-21T19:25:28.6083Z", "is_qbnewb": false, "is_superuser": true, "id": 1, "last_name": "Butler", "date_joined": "2021-07-21T05:38:53.637091Z", "common_name": "Alexander Butler"}, "can_write": true, "database_id": 2, "enable_embedding": false, "collection_id": 3, "query_type": "query", "name": "Number_of_orders over time", "dashboard_count": 1, "creator_id": 1, "updated_at": "2021-07-21T08:01:42.321381Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"type": "query", "database": 2, "query": {"source-table": 7, "breakout": [["field", 39, {"temporal-unit": "day"}]], "aggregation": [["sum", ["field", 40, null]], ["avg", ["field", 40, null]]]}}, "id": 10, "display": "line", "last-edit-info": {"id": 1, "email": "[email protected]", "first_name": "Alexander", "last_name": "Butler", "timestamp": "2021-07-21T08:01:38.368218Z"}, "visualization_settings": {"graph.series_labels": ["sum", "average"], "graph.metrics": ["sum", "avg"], "graph.dimensions": ["first_order"], "graph.colors": ["#7172AD", "#EF8C8C"]}, "collection": {"authority_level": null, "description": "Automatically generated cards.", "archived": false, "slug": "a_look_at_your_customers_table", "color": "#f9d45c", "name": "A look at your customers table", "personal_owner_id": null, "id": 3, "location": "/2/", "namespace": null}, "created_at": "2021-07-21T08:01:38.347535Z", "public_uuid": null}
+{"description": null, "archived": false, "collection_position": null, "table_id": 7, "result_metadata": [{"semantic_type": null, "coercion_strategy": null, "unit": "day", "name": "first_order", "field_ref": ["field", 39, {"temporal-unit": "day"}], "effective_type": "type/Date", "id": 39, "display_name": "first_order", "fingerprint": {"global": {"distinct-count": 47, "nil%": 0.38}, "type": {"type/DateTime": {"earliest": "2018-01-01", "latest": "2018-04-07"}}}, "base_type": "type/Date"}, {"name": "sum", "display_name": "Sum of number_of_orders", "base_type": "type/Decimal", "semantic_type": "type/Quantity", "field_ref": ["aggregation", 0], "fingerprint": {"global": {"distinct-count": 6, "nil%": 0.02127659574468085}, "type": {"type/Number": {"min": 1.0, "q1": 1.1073874870150486, "q3": 3.0, "max": 5.0, "sd": 1.2643380578934107, "avg": 2.152173913043478}}}}, {"name": "avg", "display_name": "Average of number_of_orders", "base_type": "type/Decimal", "semantic_type": "type/Quantity", "field_ref": ["aggregation", 1], "fingerprint": {"global": {"distinct-count": 8, "nil%": 0.02127659574468085}, "type": {"type/Number": {"min": 1.0, "q1": 1.019470273980893, "q3": 1.989426104361334, "max": 5.0, "sd": 0.7515349295958083, "avg": 1.596014492753623}}}}], "creator": {"email": "[email protected]", "first_name": "dbtmetabase", "last_login": "2021-07-21T19:25:28.6083Z", "is_qbnewb": false, "is_superuser": true, "id": 1, "last_name": "", "date_joined": "2021-07-21T05:38:53.637091Z", "common_name": "dbtmetabase"}, "can_write": true, "database_id": 2, "enable_embedding": false, "collection_id": 3, "query_type": "query", "name": "Number_of_orders over time", "dashboard_count": 1, "creator_id": 1, "updated_at": "2021-07-21T08:01:42.321381Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"type": "query", "database": 2, "query": {"source-table": 7, "breakout": [["field", 39, {"temporal-unit": "day"}]], "aggregation": [["sum", ["field", 40, null]], ["avg", ["field", 40, null]]]}}, "id": 10, "display": "line", "last-edit-info": {"id": 1, "email": "[email protected]", "first_name": "dbtmetabase", "last_name": "", "timestamp": "2021-07-21T08:01:38.368218Z"}, "visualization_settings": {"graph.series_labels": ["sum", "average"], "graph.metrics": ["sum", "avg"], "graph.dimensions": ["first_order"], "graph.colors": ["#7172AD", "#EF8C8C"]}, "collection": {"authority_level": null, "description": "Automatically generated cards.", "archived": false, "slug": "a_look_at_your_customers_table", "color": "#f9d45c", "name": "A look at your customers table", "personal_owner_id": null, "id": 3, "location": "/2/", "namespace": null}, "created_at": "2021-07-21T08:01:38.347535Z", "public_uuid": null}