ADAP-1118: Migrate to pyproject.toml (#1150)

* update to pyproject.toml
* install all optional dependencies in the default environment for testing
* update dagger script for hatch
* update hatch integration tests command
* keep the dagger reqs in their own file; dagger requires py310 and we test on py39
* update spark container to be more clear about what is happening when

.changes/unreleased/Under the Hood-20241207-181814.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Move from setup.py to pyproject.toml and to hatch as a dev tool
+time: 2024-12-07T18:18:14.85697-05:00
+custom:
+  Author: mikealfare
+  Issue: "1150"

.github/scripts/update_dependencies.sh

@@ -2,9 +2,9 @@
 set -e
 
 git_branch=$1
-target_req_file="dev-requirements.txt"
-core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g"
-tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g"
+target_req_file="hatch.toml"
+core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${git_branch}#subdirectory=core|g"
+tests_req_sed_pattern="s|dbt-adapters.git.*#subdirectory=dbt-tests-adapter|dbt-adapters.git@${git_branch}#subdirectory=dbt-tests-adapter|g"
 if [[ "$OSTYPE" == darwin* ]]; then
  # mac ships with a different version of sed that requires a delimiter arg
  sed -i "" "$core_req_sed_pattern" $target_req_file

.github/scripts/update_dev_dependency_branches.sh

@@ -5,8 +5,8 @@ set -e
 dbt_adapters_branch=$1
 dbt_core_branch=$2
 dbt_common_branch=$3
-target_req_file="dev-requirements.txt"
-core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g"
+target_req_file="hatch.toml"
+core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g"
 adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g"
 common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g"
 if [[ "$OSTYPE" == darwin* ]]; then

.github/workflows/integration.yml

@@ -133,13 +133,13 @@ jobs:
             ${{ inputs.dbt_adapters_branch }} \
             ${{ inputs.dbt_core_branch }} \
             ${{ inputs.dbt_common_branch }}
-          cat dev-requirements.txt
+          cat hatch.toml
+
+      - name: Install hatch
+        uses: pypa/hatch@install
 
       - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip --version
-          python -m pip install -r dagger/requirements.txt
+        run: hatch run pip install -r dagger/requirements.txt
 
       - name: Run tests for ${{ matrix.test }}
-        run: python dagger/run_dbt_spark_tests.py --profile ${{ matrix.test }}
+        run: hatch run integration-tests --profile ${{ matrix.test }}

.github/workflows/main.yml

@@ -51,20 +51,8 @@ jobs:
         with:
           python-version: '3.9'
 
-      - name: Install python dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install libsasl2-dev
-          python -m pip install --user --upgrade pip
-          python -m pip --version
-          python -m pip install pre-commit
-          pre-commit --version
-          python -m pip install -r requirements.txt
-          python -m pip install -r dev-requirements.txt
-          python -c "import dbt.adapters.spark"
-
       - name: Run pre-commit hooks
-        run: pre-commit run --all-files --show-diff-on-failure
+        uses: pre-commit/[email protected]
 
   unit:
     name: unit test / python ${{ matrix.python-version }}
@@ -87,29 +75,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install python dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install libsasl2-dev
-          python -m pip install --user --upgrade pip
-          python -m pip --version
-          python -m pip install -r requirements.txt
-          python -m pip install -r dev-requirements.txt
-          python -m pip install -e .
-
-      - name: Run unit tests
-        run: python -m pytest --color=yes --csv unit_results.csv -v tests/unit
+        uses: pypa/hatch@install
 
-      - name: Get current date
-        if: always()
-        id: date
-        run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
-
-      - uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
-          path: unit_results.csv
-          overwrite: true
+      - run: hatch run unit-tests
 
   build:
     name: build packages
@@ -128,24 +96,16 @@ jobs:
         with:
           python-version: '3.9'
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip install --upgrade setuptools wheel twine check-wheel-contents
-          python -m pip --version
+      - uses: pypa/hatch@install
 
       - name: Build distributions
-        run: ./scripts/build-dist.sh
+        run: hatch build
 
       - name: Show distributions
         run: ls -lh dist/
 
       - name: Check distribution descriptions
-        run: |
-          twine check dist/*
-      - name: Check wheel contents
-        run: |
-          check-wheel-contents dist/*.whl --ignore W007,W008
+        run: hatch run build:check-all
 
       - name: Check if this is an alpha version
         id: check-is-alpha

.github/workflows/release-internal.yml

@@ -56,17 +56,9 @@ jobs:
           python-version: "${{ env.PYTHON_TARGET_VERSION }}"
 
       - name: Install python dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install libsasl2-dev
-          python -m pip install --user --upgrade pip
-          python -m pip --version
-          python -m pip install -r requirements.txt
-          python -m pip install -r dev-requirements.txt
-          python -m pip install -e .
+        uses: pypa/hatch@install
 
-      - name: Run unit tests
-        run: python -m pytest --color=yes --csv unit_results.csv -v tests/unit
+      - run: hatch run unit-tests
 
   run-integration-tests:
     name: "${{ matrix.test }}"
@@ -102,21 +94,11 @@ jobs:
 
     steps:
       - name: Check out the repository
-        if: github.event_name != 'pull_request_target'
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-
-      # explicitly checkout the branch for the PR,
-      # this is necessary for the `pull_request` event
-      - name: Check out the repository (PR)
-        if: github.event_name == 'pull_request_target'
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-          ref: ${{ github.event.pull_request.head.ref }}
 
-      # the python version used here is not what is used in the tests themselves
+     # the python version used here is not what is used in the tests themselves
       - name: Set up Python for dagger
         uses: actions/setup-python@v5
         with:
@@ -140,6 +122,6 @@ jobs:
       package_test_command: "${{ inputs.package_test_command }}"
       dbms_name: "spark"
       ref: "${{ inputs.ref }}"
-      skip_tests: "${{ inputs.skip_tests }}"
+      skip_tests: "true"
 
     secrets: "inherit"

.github/workflows/release-prep.yml

@@ -168,8 +168,8 @@ jobs:
         run: |
           if [[ ${{ steps.set_existence.outputs.exists }} != true ]]
           then
-            title="Spark version-bump.yml check"
-            message="dbt-spark needs version-bump.yml run before running the release.  The changelog is not up to date."
+            title="Spark version bump check"
+            message="dbt-spark needs a version bump before running the release. The changelog is not up to date."
             echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
             exit 1
           fi
@@ -194,11 +194,14 @@ jobs:
         with:
           ref: ${{ inputs.sha }}
 
+      - uses: pypa/hatch@install
+
       - name: "Check Current Version In Code"
         id: version-check
         run: |
           is_updated=false
-          if grep -Fxq "current_version = ${{ inputs.version_number }}" .bumpversion.cfg
+          current_version=$(hatch version)
+          if test "$current_version" = "${{ inputs.version }}"
           then
             is_updated=true
           fi
@@ -383,26 +386,10 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_TARGET_VERSION }}
 
-      - name: "Install Python Dependencies"
-        if: needs.audit-version-in-code.outputs.up_to_date == 'false'
-        run: |
-          python3 -m venv env
-          source env/bin/activate
-          python -m pip install --upgrade pip
+      - uses: pypa/hatch@install
 
       - name: "Bump Version To ${{ inputs.version_number }}"
-        if: needs.audit-version-in-code.outputs.up_to_date == 'false'
-        # note: bumpversion is no longer supported, it actually points to bump2version now
-        run: |
-          source env/bin/activate
-          if [ -f "editable-requirements.txt" ]
-          then
-            python -m pip install -r dev-requirements.txt -r editable-requirements.txt
-          else
-            python -m pip install -r dev-requirements.txt
-          fi
-          env/bin/bumpversion --allow-dirty --new-version ${{ inputs.version_number }} major
-          git status
+        run: hatch version ${{ inputs.version_number }}
 
       - name: "[Notification] Bump Version To ${{ inputs.version_number }}"
         if: needs.audit-version-in-code.outputs.up_to_date == 'false'
@@ -415,14 +402,14 @@ jobs:
       - name: "Remove Trailing Whitespace Via Pre-commit"
         continue-on-error: true
         run: |
-          pre-commit run trailing-whitespace --files .bumpversion.cfg CHANGELOG.md .changes/*
+          pre-commit run trailing-whitespace --files CHANGELOG.md .changes/*
           git status
 
       # this step will fail on newline errors but also correct them
       - name: "Removing Extra Newlines Via Pre-commit"
         continue-on-error: true
         run: |
-          pre-commit run end-of-file-fixer --files .bumpversion.cfg CHANGELOG.md .changes/*
+          pre-commit run end-of-file-fixer --files CHANGELOG.md .changes/*
           git status
 
       - name: "Commit & Push Changes"
@@ -459,18 +446,10 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install python dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install libsasl2-dev
-          python -m pip install --user --upgrade pip
-          python -m pip --version
-          python -m pip install -r requirements.txt
-          python -m pip install -r dev-requirements.txt
-          python -m pip install -e .
+      - uses: pypa/hatch@install
 
       - name: Run unit tests
-        run: python -m pytest --color=yes --csv unit_results.csv -v tests/unit
+        run: hatch run unit-tests
 
   run-integration-tests:
     name: ${{ matrix.test }}
@@ -505,34 +484,20 @@ jobs:
 
     steps:
       - name: Check out the repository
-        if: github.event_name != 'pull_request_target'
         uses: actions/checkout@v4
         with:
           persist-credentials: false
 
-      # explicitly checkout the branch for the PR,
-      # this is necessary for the `pull_request` event
-      - name: Check out the repository (PR)
-        if: github.event_name == 'pull_request_target'
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-          ref: ${{ github.event.pull_request.head.sha }}
-
       # the python version used here is not what is used in the tests themselves
       - name: Set up Python for dagger
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip --version
-          python -m pip install -r dagger/requirements.txt
+      - uses: pypa/hatch@install
 
       - name: Run tests for ${{ matrix.test }}
-        run: python dagger/run_dbt_spark_tests.py --profile ${{ matrix.test }}
+        run: hatch run python dagger/run_dbt_spark_tests.py --profile ${{ matrix.test }}
 
   merge-changes-into-target-branch:
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -54,5 +54,7 @@ repos:
         -   --pretty
         files: ^dbt/adapters
         additional_dependencies:
+        -   types-PyYAML
+        -   types-python-dateutil
         -   types-pytz
         -   types-requests