Skip to content

Commit

Permalink
ADAP-1117: Move to hatch and pyproject.toml (#951)
Browse files Browse the repository at this point in the history
* move config into pyproject.toml or hatch.toml
  • Loading branch information
mikealfare authored Dec 11, 2024
1 parent 99d6a3d commit a145bc1
Show file tree
Hide file tree
Showing 17 changed files with 641 additions and 423 deletions.
35 changes: 0 additions & 35 deletions .bumpversion.cfg

This file was deleted.

6 changes: 6 additions & 0 deletions .changes/unreleased/Under the Hood-20241207-165918.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Under the Hood
body: Move from setup.py to pyproject.toml and to hatch as a dev tool
time: 2024-12-07T16:59:18.731819-05:00
custom:
Author: mikealfare
Issue: "951"
8 changes: 4 additions & 4 deletions .github/scripts/update_dependencies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
set -e

git_branch=$1
target_req_file="dev-requirements.txt"
core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g"
postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g"
tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g"
target_req_file="hatch.toml"
core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${git_branch}#subdirectory=core|g"
postgres_req_sed_pattern="s|dbt-postgres.git|dbt-postgres.git@${git_branch}"
tests_req_sed_pattern="s|dbt-adapters.git.*#subdirectory=dbt-tests-adapter|dbt-adapters.git@${git_branch}#subdirectory=dbt-tests-adapter|g"
if [[ "$OSTYPE" == darwin* ]]; then
# mac ships with a different version of sed that requires a delimiter arg
sed -i "" "$core_req_sed_pattern" $target_req_file
Expand Down
4 changes: 2 additions & 2 deletions .github/scripts/update_dev_dependency_branches.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ set -e
dbt_adapters_branch=$1
dbt_core_branch=$2
dbt_common_branch=$3
target_req_file="dev-requirements.txt"
core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g"
target_req_file="hatch.toml"
core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g"
adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g"
common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g"
if [[ "$OSTYPE" == darwin* ]]; then
Expand Down
46 changes: 6 additions & 40 deletions .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,6 @@ jobs:
python-version: "3.9"

env:
TOXENV: integration-redshift
PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
DBT_INVOCATION_ENV: github-actions
DD_CIVISIBILITY_AGENTLESS_ENABLED: true
DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
Expand Down Expand Up @@ -144,14 +142,9 @@ jobs:
${{ inputs.dbt_adapters_branch }} \
${{ inputs.dbt_core_branch }} \
${{ inputs.dbt_common_branch }}
cat dev-requirements.txt
cat hatch.toml
- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install tox
python -m pip --version
tox --version
- uses: pypa/hatch@install

- name: Create AWS IAM profiles
run: |
Expand Down Expand Up @@ -180,7 +173,7 @@ jobs:
AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }}
AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }}

- name: Run tox (redshift)
- run: hatch run integration-tests tests/functional -m "not flaky"
env:
REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
Expand All @@ -196,25 +189,6 @@ jobs:
DBT_TEST_USER_1: dbt_test_user_1
DBT_TEST_USER_2: dbt_test_user_2
DBT_TEST_USER_3: dbt_test_user_3
run: tox -- -m "not flaky" --ddtrace

- uses: actions/upload-artifact@v3
if: always()
with:
name: logs
path: ./logs

- name: Get current date
if: always()
id: date
run: |
echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
- uses: actions/upload-artifact@v3
if: always()
with:
name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_redshift-${{ steps.date.outputs.date }}.csv
path: integration_results.csv

test-flaky:
name: redshift / python ${{ matrix.python-version }} / ubuntu-22.04 - flaky
Expand All @@ -236,8 +210,6 @@ jobs:
python-version: ["3.9", "3.10", "3.11", "3.12"]

env:
TOXENV: integration-redshift
PYTEST_ADDOPTS: "-v --color=yes -n1 --csv integration_results.csv"
DBT_INVOCATION_ENV: github-actions
DD_CIVISIBILITY_AGENTLESS_ENABLED: true
DD_INSTRUMENTATION_TELEMETRY_ENABLED: false
Expand Down Expand Up @@ -274,12 +246,7 @@ jobs:
with:
python-version: ${{ matrix.python-version }}

- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install tox
python -m pip --version
tox --version
- uses: pypa/hatch@install

- name: Update Adapters and Core branches (update dev_requirements.txt)
if: github.event_name == 'workflow_dispatch'
Expand All @@ -288,8 +255,7 @@ jobs:
${{ inputs.dbt_adapters_branch }} \
${{ inputs.dbt_core_branch }} \
${{ inputs.dbt_common_branch }}
cat dev-requirements.txt
cat hatch.toml
- name: Run tox (redshift)
env:
Expand All @@ -307,7 +273,7 @@ jobs:
DBT_TEST_USER_1: dbt_test_user_1
DBT_TEST_USER_2: dbt_test_user_2
DBT_TEST_USER_3: dbt_test_user_3
run: tox -- -m flaky --ddtrace
run: hatch run integration-tests tests/functional -m flaky -n1 --ddtrace

require-label-comment:
runs-on: ubuntu-22.04
Expand Down
74 changes: 12 additions & 62 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,19 +48,12 @@ jobs:
persist-credentials: false

- name: Set up Python
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: '3.9'

- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install -r dev-requirements.txt
python -m pip --version
pre-commit --version
- name: pre-commit hooks
run: pre-commit run --all-files --show-diff-on-failure
uses: pre-commit/[email protected]

unit:
name: unit test / python ${{ matrix.python-version }}
Expand All @@ -72,10 +65,6 @@ jobs:
matrix:
python-version: ['3.9', '3.10', '3.11', '3.12']

env:
TOXENV: "unit"
PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv"

steps:
- name: Check out the repository
uses: actions/checkout@v4
Expand All @@ -88,27 +77,9 @@ jobs:
python-version: ${{ matrix.python-version }}

- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install tox
python -m pip --version
tox --version
- name: Run tox
run: tox

- name: Get current date
if: always()
id: date
#no colons allowed for artifacts
run: |
echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT
uses: pypa/hatch@install

- uses: actions/upload-artifact@v3
if: always()
with:
name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
path: unit_results.csv
- run: hatch run unit-tests

build:
name: build packages
Expand All @@ -130,24 +101,16 @@ jobs:
python-version: '3.9'

- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install --upgrade setuptools wheel twine check-wheel-contents
python -m pip --version
uses: pypa/hatch@install

- name: Build distributions
run: ./scripts/build-dist.sh
run: hatch build

- name: Show distributions
run: ls -lh dist/

- name: Check distribution descriptions
run: |
twine check dist/*
- name: Check wheel contents
run: |
check-wheel-contents dist/*.whl --ignore W007,W008
run: hatch run build:check-all

- name: Check if this is an alpha version
id: check-is-alpha
Expand Down Expand Up @@ -175,6 +138,7 @@ jobs:
matrix:
os: [ubuntu-22.04, macos-14, windows-2022]
python-version: ['3.9', '3.10', '3.11', '3.12']
dist-type: ["whl", "gz"]
exclude:
# psycopg2-binary doesn't have a precompiled wheel for python 3.9 for mac
- os: macos-14
Expand All @@ -186,12 +150,6 @@ jobs:
with:
python-version: ${{ matrix.python-version }}

- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install --upgrade wheel
python -m pip --version
- uses: actions/download-artifact@v3
with:
name: dist
Expand All @@ -200,18 +158,10 @@ jobs:
- name: Show distributions
run: ls -lh dist/

- name: Install wheel distributions
run: |
find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
- name: Check wheel distributions
run: |
python -c "import dbt.adapters.redshift"
- name: Install source distributions
- name: Install ${{ matrix.dist-type }} distributions
run: |
find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
- name: Check source distributions
- name: Check ${{ matrix.dist-type }} distributions
run: |
python -c "import dbt.adapters.redshift"
python -c "import dbt.adapters.bigquery"
14 changes: 9 additions & 5 deletions .github/workflows/nightly-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,15 @@ jobs:
commit_sha=$(git rev-parse HEAD)
echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT
- name: "Get Current Version Number"
id: version-number-sources
run: |
current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '`
echo "current_version=$current_version" >> $GITHUB_OUTPUT
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.9'

- uses: pypa/hatch@install

- id: version-number-sources
run: echo "current_version=$(hatch version)" >> $GITHUB_OUTPUT

- name: "Audit Version And Parse Into Parts"
id: semver
Expand Down
Loading

0 comments on commit a145bc1

Please sign in to comment.