Merge master changes

Datavault-UK · Mar 27, 2021 · 694c616 · 694c616
2 parents 73c2166 + ad08bca
commit 694c616
Show file tree

Hide file tree

Showing 7 changed files with 110 additions and 14 deletions.
diff --git a/dbtvault-dev/.circleci/config.yml b/dbtvault-dev/.circleci/config.yml
@@ -0,0 +1,96 @@
+version: 2.1
+
+
+orbs:
+  slack: circleci/[email protected]
+  secrethub: secrethub/[email protected]
+
+commands:
+  build_test_env:
+    description: "Build the test environment"
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-dbtvault-dev-{{ arch }}-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
+            - v1-dbtvault-dev-{{ arch }}-{{ .Branch }}
+            - v1-dbtvault-dev-
+      - run:
+          name: Install dependencies
+          command: |
+            pipenv install --dev
+            pipenv install
+      - save_cache:
+          key: v1-dbtvault-dev-{{ arch }}-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
+          paths:
+            - /.circleci/.cache
+      - secrethub/install
+      - run:
+          name: Install dbt dependencies in test project
+          command: TARGET=snowflake pipenv run inv run-dbt -u circleci -t snowflake -p test -d 'deps' -e secrethub/secrethub_circleci.env
+
+jobs:
+  macros:
+    docker:
+      - image: cimg/python:3.8.5
+    parallelism: 10
+    steps:
+      - build_test_env
+      - run:
+          name: Run snowflake macro tests
+          command: |
+            circleci tests glob test_project/unit/*/test_*.py | circleci tests split > /tmp/macro-tests-to-run
+            TARGET=snowflake pipenv run inv macro-tests -t snowflake -u circleci -e secrethub/secrethub_circleci.env
+      - slack/status:
+          fail_only: false
+          mentions: 'URXTX0XEZ'
+      - store_test_results:
+          path: test_results/integration_tests
+      - store_test_results:
+          path: test_results/macro_tests
+      - store_artifacts:
+          path: test_results/integration_tests
+      - store_artifacts:
+          path: test_results/macro_tests
+
+  integration:
+    docker:
+      - image: cimg/python:3.8.5
+    parallelism: 15
+    steps:
+      - build_test_env
+      - run:
+          name: Run snowflake integration tests
+          command: |
+            circleci tests glob test_project/features/*/*.feature | circleci tests split > /tmp/feature-tests-to-run
+            TARGET=snowflake pipenv run inv integration-tests -t snowflake -u circleci -e secrethub/secrethub_circleci.env
+      - slack/status:
+          fail_only: false
+          mentions: 'URXTX0XEZ'
+      - store_test_results:
+          path: test_results/integration_tests
+      - store_test_results:
+          path: test_results/macro_tests
+      - store_artifacts:
+          path: test_results/integration_tests
+      - store_artifacts:
+          path: test_results/macro_tests
+
+workflows:
+  version: 2
+  test-macros:
+    jobs:
+      - macros:
+          filters:
+            branches:
+              only:
+                - develop
+                - /^int.*/
+  test-integration:
+    jobs:
+      - integration:
+          filters:
+            branches:
+              only:
+                - develop
+                - /^int.*/
diff --git a/dbtvault-dev/.github/ISSUE_TEMPLATE/bug_report.md b/dbtvault-dev/.github/ISSUE_TEMPLATE/bug_report.md
@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve
 title: "[BUG] "
-labels: ''
+labels: bug
 assignees: DVAlexHiggs
 
 ---

diff --git a/dbtvault-dev/.github/ISSUE_TEMPLATE/feature_request.md b/dbtvault-dev/.github/ISSUE_TEMPLATE/feature_request.md
@@ -2,7 +2,7 @@
 name: Feature request
 about: Suggest an idea for this project
 title: "[FEATURE]"
-labels: ''
+labels: feature
 assignees: DVAlexHiggs
 
 ---

diff --git a/dbtvault-dev/.gitignore b/dbtvault-dev/.gitignore
@@ -0,0 +1,2 @@
+
+.idea/
diff --git a/dbtvault-dev/README.md b/dbtvault-dev/README.md
@@ -11,7 +11,6 @@
     src="https://img.shields.io/badge/Slack-Join-yellow?style=flat&logo=slack" 
     alt="Join our slack"
   /></a>
-
 </p>
 
 
@@ -54,10 +53,16 @@ or [read the docs](https://docs.getdbt.com/docs/building-a-dbt-project/package-m
 3. Call the appropriate template macro
 
 ```bash
-{{- config(...)                                                           -}}
+# Configure model
+{{- config(...)                          -}}
 
-{{ dbtvault.hub(var('src_pk'), var('src_nk'), var('src_ldts'),
-                var('src_source'), var('source_model'))                    }}
+# Set metadata
+{%- set src_pk = ...                     -%}
+...
+
+# Call the macro
+{{ dbtvault.hub(src_pk, src_nk, src_ldts,
+                src_source, source_model) }}
 ```
 
 ## Join our Slack Channel

diff --git a/dbtvault-dev/dbt_project.yml b/dbtvault-dev/dbt_project.yml
@@ -2,7 +2,6 @@ name: 'dbtvault'
 version: '0.7.3'
 require-dbt-version: [">=0.18.0", "<0.20.0"]
 config-version: 2
-profile: dbtvault
 
 source-paths: ["models"]
 analysis-paths: ["analysis"] 

diff --git a/dbtvault-dev/macros/staging/stage.sql b/dbtvault-dev/macros/staging/stage.sql
@@ -25,13 +25,7 @@
     OR
     [SOURCES STYLE]
     source_model:
-        source_name: source_table_name
-
-    Please check:
-    - If using variables with var(), that your variable is scoped/provided correctly.
-    - That you are only providing the metadata as shown above. dbtvault converts this to
-      ref() or source() internally, you only need to provide strings!
-
+        source_name: source_table_name"
     {%- endset -%}
 
     {{- exceptions.raise_compiler_error(error_message) -}}