Clean up #25
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Perform deployment to production | |
| on: # yamllint disable-line rule:truthy | |
| pull_request: | |
| types: | |
| - closed | |
| branches: | |
| - main | |
| paths: | |
| - .github/workflows/* | |
| - automate/* | |
| - transform/* | |
| - transform/**/* | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - .github/workflows/* | |
| - automate/** | |
| - transform/* | |
| - transform/**/* | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| jobs: | |
| # Perform the deployment to Prod | |
| build: | |
| # Need to make sure that when the PR was closed, it was actually merged. | |
| # if: github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main' | |
| name: Deployment Script | |
| runs-on: ubuntu-latest | |
| # Set environment variables in | |
| # https://github.com//<your org>/<your repo>/settings/variables/actions | |
| # | |
| # Alternatively, You can define multiple ENV for different workflows. | |
| # https://github.com/<org>/<repo>/settings/environments | |
| # environment: PR_ENV | |
| container: datacoves/ci-basic-dbt-snowflake:3.2 | |
| defaults: | |
| run: | |
| working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform | |
| env: | |
| DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/automate/dbt | |
| DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform | |
| DATACOVES__MAIN__ACCOUNT: ${{ vars.DATACOVES__MAIN__ACCOUNT }} | |
| DATACOVES__MAIN__DATABASE: ${{ vars.DATACOVES__MAIN__DATABASE }} | |
| DATACOVES__MAIN__SCHEMA: ${{ vars.DATACOVES__MAIN__SCHEMA }} | |
| DATACOVES__MAIN__ROLE: ${{ vars.DATACOVES__MAIN__ROLE }} | |
| DATACOVES__MAIN__WAREHOUSE: ${{ vars.DATACOVES__MAIN__WAREHOUSE }} | |
| DATACOVES__MAIN__USER: ${{ vars.DATACOVES__MAIN__USER }} | |
| DATACOVES__MAIN__PASSWORD: ${{ secrets.DATACOVES__MAIN__PASSWORD }} | |
| # This is used by datacoves to drop the staging database for blue/green | |
| # deployments, most likely you don't want to set this, we use it for demos | |
| DATACOVES__DROP_DB_ON_FAIL: ${{ vars.DATACOVES__DROP_DB_ON_FAIL }} | |
| steps: | |
| - name: Checkout branch | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ github.event.push.head.sha }} | |
| fetch-depth: 0 | |
| - name: Set Secure Directory | |
| run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} | |
| - name: Install dbt packages | |
| run: "dbt deps" | |
| - name: Get prod manifest | |
| id: prod-manifest | |
| run: "../automate/dbt/get_artifacts.sh" | |
| # Runs blue green with no deferral | |
| - name: Run blue / green deployment | |
| if: ${{ steps.prod-manifest.outputs.manifest_found == 'false' }} | |
| id: run-blue-green | |
| run: "dbt-coves blue-green" | |
| # Runs blue green with deferral | |
| - name: Run blue / green deployment with deferral | |
| if: ${{ steps.prod-manifest.outputs.manifest_found == 'true' }} | |
| id: run-blue-green-defer | |
| run: "dbt-coves blue-green --defer" | |
| - name: Drop orphaned relations in db that are no longer in dbt | |
| run: "dbt run-operation drop_orphaned_relations --args '{\"dry_run\": false}'" | |
| - name: Generate dbt documentation | |
| run: "dbt docs generate" | |
| - name: Deploy docs 🚀 | |
| uses: JamesIves/[email protected] | |
| with: | |
| branch: dbt-docs | |
| folder: transform/target | |
| - name: Upload dbt artifacts | |
| run: "dbt run-operation upload_artifacts" | |
| # Drops the temporary PR database | |
| drop-pr-db-on-close: | |
| name: Drop PR Database on Close | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| # Set environment variables in | |
| # https://github.com//<your org>/<your repo>/settings/variables/actions | |
| # | |
| # Alternatively, You can define multiple ENV for different workflows. | |
| # https://github.com/<org>/<repo>/settings/environments | |
| # environment: PR_ENV | |
| container: datacoves/ci-basic-dbt-snowflake:3.2 | |
| defaults: | |
| run: | |
| working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform | |
| env: | |
| DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/automate/dbt | |
| DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform | |
| DATACOVES__YAML_DAGS_FOLDER: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/schedule | |
| DATACOVES__MAIN__ACCOUNT: ${{ vars.DATACOVES__MAIN__ACCOUNT }} | |
| DATACOVES__MAIN__DATABASE: ${{ vars.DATACOVES__MAIN__DATABASE }}_PR_${{ github.event.number }} | |
| DATACOVES__MAIN__SCHEMA: ${{ vars.DATACOVES__MAIN__SCHEMA }} | |
| DATACOVES__MAIN__ROLE: ${{ vars.DATACOVES__MAIN__ROLE }} | |
| DATACOVES__MAIN__WAREHOUSE: ${{ vars.DATACOVES__MAIN__WAREHOUSE }} | |
| DATACOVES__MAIN__USER: ${{ vars.DATACOVES__MAIN__USER }} | |
| DATACOVES__MAIN__PASSWORD: ${{ secrets.DATACOVES__MAIN__PASSWORD }} | |
| steps: | |
| - name: Checkout branch | |
| uses: actions/[email protected] | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set Secure Directory | |
| run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} | |
| - name: Install dbt packages | |
| run: "dbt deps" | |
| - name: Drop PR database | |
| run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{env.DATACOVES__MAIN__DATABASE}}, recreate: False}'" # yamllint disable-line rule:line-length |