From e5ae13c881e92a77c81288e479f6699b1e0b7fc3 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Jul 2024 21:31:19 -0400 Subject: [PATCH 1/8] collapse hatch environments into the default environment --- pyproject.toml | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 92fbf82a..a7294f7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,12 +56,25 @@ path = "dbt/adapters/postgres/__version__.py" dependencies = [ "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", 'pre-commit==3.7.0;python_version>="3.9"', 'pre-commit==3.5.0;python_version=="3.8"', + "freezegun", + "pytest", + "pytest-dotenv", + "pytest-mock", + "pytest-xdist", ] +[tool.hatch.envs.default.env-vars] +DBT_TEST_USER_1 = "dbt_test_user_1" +DBT_TEST_USER_2 = "dbt_test_user_2" +DBT_TEST_USER_3 = "dbt_test_user_3" [tool.hatch.envs.default.scripts] dev = "pre-commit install" code-quality = "pre-commit run --all-files" +unit-tests = "python -m pytest {args:tests/unit}" +integration-tests = "python -m pytest {args:tests/functional}" docker-dev = [ "echo Does not support integration testing, only development and unit testing. See issue https://github.com/dbt-labs/dbt-postgres/issues/99", "docker build -f docker/dev.Dockerfile -t dbt-postgres-dev .", @@ -69,32 +82,6 @@ docker-dev = [ ] docker-prod = "docker build -f docker/Dockerfile -t dbt-postgres ." -[tool.hatch.envs.unit-tests] -dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", - "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", - "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", - "freezegun", - "pytest", - "pytest-dotenv", - "pytest-mock", - "pytest-xdist", -] -[tool.hatch.envs.unit-tests.scripts] -all = "python -m pytest {args:tests/unit}" - -[tool.hatch.envs.integration-tests] -template = "unit-tests" -extra-dependencies = [ - "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", -] -[tool.hatch.envs.integration-tests.env-vars] -DBT_TEST_USER_1 = "dbt_test_user_1" -DBT_TEST_USER_2 = "dbt_test_user_2" -DBT_TEST_USER_3 = "dbt_test_user_3" -[tool.hatch.envs.integration-tests.scripts] -all = "python -m pytest {args:tests/functional}" - [tool.hatch.envs.build] detached = true dependencies = [ From 3b5513e171e1d3e4a6db67c780c9265e67b9c24a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Jul 2024 21:33:11 -0400 Subject: [PATCH 2/8] update workflows for new hatch envs --- .github/workflows/integration-tests.yml | 2 +- .github/workflows/release_prep_hatch.yml | 4 ++-- .github/workflows/unit-tests.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d7367190..e3390fbd 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -91,7 +91,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run integration tests - run: hatch run integration-tests:all + run: hatch run integration-tests env: POSTGRES_TEST_HOST: localhost POSTGRES_TEST_PORT: 5432 diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index 8e9ded04..4b89d343 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -342,7 +342,7 @@ jobs: uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - name: "Run unit tests" - run: hatch run unit-tests:all + run: hatch run unit-tests integration-tests: runs-on: ubuntu-latest @@ -387,7 +387,7 @@ jobs: uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - name: "Run integration tests" - run: hatch run integration-tests:all + run: hatch run integration-tests env: POSTGRES_TEST_HOST: localhost POSTGRES_TEST_PORT: 5432 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 91d8c578..e171ff78 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -37,5 +37,5 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run unit tests - run: hatch run unit-tests:all + run: hatch run unit-tests shell: bash From 6148ea1292fafb411cbb2998addbfc8b9dc7ba21 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Jul 2024 21:48:45 -0400 Subject: [PATCH 3/8] update the contributing guide with the new hatch setup --- CONTRIBUTING.md | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 427d2de5..deb93fc1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,21 +66,18 @@ Rather than forking `dbt-labs/dbt-postgres`, use `dbt-labs/dbt-postgres` directl ### Installation -1. Ensure the latest version of `pip` is installed: +1. Ensure the latest versions of `pip` and `hatch` are installed: ```shell - pip install --upgrade pip + pip install --upgrade pip hatch ``` -2. Configure and activate a virtual environment using `virtualenv` as described in -[Setting up an environment](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md#setting-up-an-environment) -3. Install `dbt-postgres` and development dependencies in the virtual environment +2. Create a `hatch` environment with all of the development dependencies and activate it: ```shell - pip install -e .[dev] + hatch shell ``` When `dbt-postgres` is installed this way, any changes made to the `dbt-postgres` source code will be reflected in the virtual environment immediately. - ## Testing `dbt-postgres` contains [unit](https://github.com/dbt-labs/dbt-postgres/tree/main/tests/unit) @@ -95,9 +92,9 @@ Unit tests can be run locally without setting up a database connection: # Note: replace $strings with valid names # run all unit tests in a module -python -m pytest tests/unit/$test_file_name.py +hatch run unit-tests tests/unit/$test_file_name.py # run a specific unit test -python -m pytest tests/unit/$test_file_name.py::$test_class_name::$test_method_name +hatch run unit-tests tests/unit/$test_file_name.py::$test_class_name::$test_method_name ``` ### Functional tests @@ -120,17 +117,22 @@ Functional tests can be run locally with a valid database connection configured ```shell # Note: replace $strings with valid names +# run all functional tests +hatch run integration-tests + # run all functional tests in a directory -python -m pytest tests/functional/$test_directory +hatch run integration-tests tests/functional/$test_directory + # run all functional tests in a module -python -m pytest tests/functional/$test_dir_and_filename.py +hatch run integration-tests tests/functional/$test_directory/$test_filename.py + # run all functional tests in a class -python -m pytest tests/functional/$test_dir_and_filename.py::$test_class_name +hatch run integration-tests tests/functional/$test_directory/$test_filename.py::$test_class_name + # run a specific functional test -python -m pytest tests/functional/$test_dir_and_filename.py::$test_class_name::$test__method_name +hatch run integration-tests tests/functional/$test_directory/$test_filename.py::$test_class_name::$test__method_name ``` - ## Documentation ### User documentation From 083cb404dab4377b5249a7ed7ed1d42205be5ac7 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Jul 2024 22:06:07 -0400 Subject: [PATCH 4/8] use a more intuitive name for venv creation --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a7294f7b..5edecc9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ DBT_TEST_USER_1 = "dbt_test_user_1" DBT_TEST_USER_2 = "dbt_test_user_2" DBT_TEST_USER_3 = "dbt_test_user_3" [tool.hatch.envs.default.scripts] -dev = "pre-commit install" +setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" integration-tests = "python -m pytest {args:tests/functional}" From 8842c821ba91fb97f9220b1da9f66bfbc3760efa Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Jul 2024 22:06:32 -0400 Subject: [PATCH 5/8] use a more intuitive name for venv creation --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index deb93fc1..06ddb77a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,6 +72,7 @@ Rather than forking `dbt-labs/dbt-postgres`, use `dbt-labs/dbt-postgres` directl ``` 2. Create a `hatch` environment with all of the development dependencies and activate it: ```shell + hatch run setup hatch shell ``` From e5390b514c6b5d7b0c998ed02734b01434a18ab2 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Jul 2024 22:36:45 -0400 Subject: [PATCH 6/8] update the contributing guide --- CONTRIBUTING.md | 45 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 06ddb77a..9761dea6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,22 +68,40 @@ Rather than forking `dbt-labs/dbt-postgres`, use `dbt-labs/dbt-postgres` directl 1. Ensure the latest versions of `pip` and `hatch` are installed: ```shell - pip install --upgrade pip hatch + pip install --user --upgrade pip hatch ``` -2. Create a `hatch` environment with all of the development dependencies and activate it: +2. This step is optional, but it's recommended. Configure `hatch` to create its virtual environments in the project. Add this block to your `hatch` `config.toml` file: + ```toml + # MacOS: ~/Library/Application Support/hatch/config.toml + [dirs.env] + virtual = ".hatch" + ``` + This makes `hatch` create all virtual environments in the project root inside of the directory `/.hatch`, similar to `/.tox` for `tox`. + It also makes it easier to add this environment as a runner in common IDEs like VSCode and PyCharm. +3. Create a `hatch` environment with all of the development dependencies and activate it: ```shell hatch run setup hatch shell ``` +4. Run any commands within the virtual environment by prefixing the command with `hatch run`: + ```shell + hatch run + ``` When `dbt-postgres` is installed this way, any changes made to the `dbt-postgres` source code will be reflected in the virtual environment immediately. ## Testing -`dbt-postgres` contains [unit](https://github.com/dbt-labs/dbt-postgres/tree/main/tests/unit) -and [functional](https://github.com/dbt-labs/dbt-postgres/tree/main/tests/functional) tests. +`dbt-postgres` contains [code quality checks](https://github.com/dbt-labs/dbt-postgres/tree/main/.pre-commit-config.yaml), [unit tests](https://github.com/dbt-labs/dbt-postgres/tree/main/tests/unit), +and [functional tests](https://github.com/dbt-labs/dbt-postgres/tree/main/tests/functional). +### Code quality + +Code quality checks can run with a single command: +```shell +hatch run code-quality +``` ### Unit tests @@ -92,8 +110,12 @@ Unit tests can be run locally without setting up a database connection: ```shell # Note: replace $strings with valid names +# run all unit tests +hatch run unit-test + # run all unit tests in a module hatch run unit-tests tests/unit/$test_file_name.py + # run a specific unit test hatch run unit-tests tests/unit/$test_file_name.py::$test_class_name::$test_method_name ``` @@ -134,6 +156,21 @@ hatch run integration-tests tests/functional/$test_directory/$test_filename.py:: hatch run integration-tests tests/functional/$test_directory/$test_filename.py::$test_class_name::$test__method_name ``` +### Testing against a development branch + +Some changes require a change in `dbt-common` and/or `dbt-adapters`. +In that case, the dependency on `dbt-common` and/or `dbt-adapters` must be updated to point to the development branch. For example: + +```toml +[tool.hatch.envs.default] +dependencies = [ + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git@my-dev-branch", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@my-dev-branch", + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git@my-dev-branch#subdirectory=dbt-tests-adapter", + ..., +] +``` + ## Documentation ### User documentation From cb614d338b9ee3e46c2d99fdbc90115a7ce6df25 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Jul 2024 22:58:52 -0400 Subject: [PATCH 7/8] update the contributing guide for cross repo features --- CONTRIBUTING.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9761dea6..9d925156 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -171,6 +171,15 @@ dependencies = [ ] ``` +This will install `dbt-common`/`dbt-adapters`/`dbt-tests-adapter` as snapshots. In other words, if `my-dev-branch` is updated on GitHub, those updates will not be reflected locally. +In order to pick up those updates, the `hatch` environment(s) will need to be rebuilt: + +```shell +exit +hatch env prune +hatch shell +``` + ## Documentation ### User documentation From 9c6d22084a7987bd9c47eb8352aea125af616c51 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 16 Jul 2024 16:08:45 -0400 Subject: [PATCH 8/8] fix broken references --- .github/workflows/release_prep_hatch.yml | 6 +++--- pyproject.toml | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index 4b89d343..ab343b45 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -31,7 +31,7 @@ # 1. Bump the version if it has not been bumped # 2. Generate the changelog (via changie) if there is no markdown file for this version name: "Release prep" -run-name: "Release prep: Generate changelog and bump ${{ inputs.package }} to ${{ inputs.version }} for release to ${{ inputs.deploy-to }}" +run-name: "Release prep: Generate changelog and bump to ${{ inputs.version }} for release to ${{ inputs.deploy-to }}" on: workflow_call: inputs: @@ -464,6 +464,6 @@ jobs: run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT # if this is a real release and a release branch was created, delete it - - name: "Delete release branch: ${{ needs.branch.outputs.name }}" + - name: "Delete release branch: ${{ needs.release-branch.outputs.name }}" if: ${{ inputs.deploy-to == 'prod' && inputs.is-nightly-release == 'false' && needs.release-branch.outputs.name != '' }} - run: git push origin -d ${{ needs.branch.outputs.name }} + run: git push origin -d ${{ needs.release-branch.outputs.name }} diff --git a/pyproject.toml b/pyproject.toml index 5edecc9f..c6454835 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ dependencies = [ "dbt-common>=0.1.0a1,<2.0", "agate>=1.0,<2.0", ] - [project.urls] Homepage = "https://github.com/dbt-labs/dbt-postgres" Documentation = "https://docs.getdbt.com"