Skip to content

Commit

Permalink
use nox everywhere for testing
Browse files Browse the repository at this point in the history
  • Loading branch information
jhkennedy committed Nov 6, 2024
1 parent 50bf0b9 commit c5ba49b
Show file tree
Hide file tree
Showing 7 changed files with 45 additions and 56 deletions.
24 changes: 0 additions & 24 deletions .github/actions/install-pkg/action.yml

This file was deleted.

8 changes: 4 additions & 4 deletions .github/workflows/integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -94,16 +94,16 @@ jobs:
# this point.
ref: ${{ github.event.pull_request.head.sha }}

- name: Install package with dependencies
uses: ./.github/actions/install-pkg
- name: Setup nox
uses: wntrblm/[email protected]
with:
python-version: ${{ matrix.python-version }}
python-versions: ${{ matrix.python-version }}

- name: Run integration tests
env:
EARTHDATA_USERNAME: ${{ secrets.EDL_USERNAME }}
EARTHDATA_PASSWORD: ${{ secrets.EDL_PASSWORD }}
run: ./scripts/integration-test.sh
run: nox -s integration-tests -- --cov=earthaccess --cov-report=term-missing --capture=no --tb=native --log-cli-level=INFO

- name: Upload coverage report
# Don't upload coverage when using the `act` tool to run the workflow locally
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/test-mindeps.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ jobs:
version: "0.4.7"
enable-cache: true

- name: Set up Python
uses: actions/setup-python@v5
- name: Setup nox
uses: wntrblm/[email protected]
with:
python-version: 3.9

Expand All @@ -39,7 +39,7 @@ jobs:
run: uv pip install --no-deps .

- name: Test
run: uv run pytest tests/unit --cov=earthaccess --cov=tests --cov-report=term-missing --capture=no --tb=native --log-cli-level=INFO
run: nox -s test-min-deps -- --verbose --cov=earthaccess --cov-report=term-missing --capture=no --tb=native --log-cli-level=INFO

- name: Upload coverage
# Don't upload coverage when using the `act` tool to run the workflow locally
Expand Down
9 changes: 5 additions & 4 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,16 @@ jobs:
- name: Checkout sources
uses: actions/checkout@v4

- uses: ./.github/actions/install-pkg
- name: Setup nox
uses: wntrblm/[email protected]
with:
python-version: ${{ matrix.python-version }}
python-versions: ${{ matrix.python-version }}

- name: Typecheck
run: mypy
run: nox -s typecheck

- name: Test
run: pytest tests/unit --verbose --cov=earthaccess --cov-report=term-missing --capture=no --tb=native --log-cli-level=INFO
run: nox -s tests -- --verbose --cov=earthaccess --cov-report=term-missing --capture=no --tb=native --log-cli-level=INFO

- name: Upload coverage
# Don't upload coverage when using the `act` tool to run the workflow locally
Expand Down
26 changes: 22 additions & 4 deletions noxfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,27 +27,45 @@ def tests(session: nox.Session) -> None:
session.run(
"pytest",
"tests/unit",
"-rxXs", # Show provided reason in summary for (x)fail, (X)pass, and (s)kipped statuses
"-rxXs", # Show provided reason in summary for (x)fail, (X)pass, and (s)kipped tests
*session.posargs,
)

@nox.session(name="test-min-deps", python="3.9")
def test_min_deps(session: nox.Session) -> None:
"""Run the unit tests."""
session.install("--resolution", "lowest-direct", "--editable", ".[test]")
session.run(
"pytest",
"tests/unit",
"-rxXs", # Show provided reason in summary for (x)fail, (X)pass, and (s)kipped tests
*session.posargs,
)



@nox.session(name="integration-tests")
def integration_tests(session: nox.Session) -> None:
"""Run the integration tests."""
session.install("--editable", ".[test]")
session.run(
"scripts/integration-test.sh",
"pytest",
"tests/integration",
"-rxXs", # Show provided reason in summary for (x)fail, (X)pass, and (s)kipped tests
*session.posargs,
env=dict(
EARTHDATA_USERNAME=os.environ["EARTHDATA_USERNAME"],
EARTHDATA_PASSWORD=os.environ["EARTHDATA_PASSWORD"],
),
external=True,
# NOTE: integration test are permitted to pass if the failure rate was less than a hardcoded threshold.
# PyTest will return 99 if there were some failures, but less than the threshold. For more details, see:
# `pytest_sessionfinish` in tests/integration/conftest.py
success_codes=[0, 99],
)


@nox.session
@nox.session(name="build-pkg")
def build_pkg(session: nox.Session) -> None:
"""Build a source distribution and binary distribution (wheel)."""
build_path = DIR.joinpath("build")
Expand All @@ -58,7 +76,7 @@ def build_pkg(session: nox.Session) -> None:
session.run("python", "-m", "build")


@nox.session
@nox.session(name="serve-docs")
def serve_docs(session: nox.Session) -> None:
"""Build the documentation and serve it."""
session.install("--editable", ".[docs]")
Expand Down
16 changes: 0 additions & 16 deletions scripts/integration-test.sh

This file was deleted.

12 changes: 11 additions & 1 deletion tests/integration/conftest.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import os
import pathlib
from warnings import warn

import earthaccess
import pytest
Expand All @@ -22,6 +23,9 @@ def pytest_sessionfinish(session, exitstatus):
ratio will change depending on which tests are executed! E.g. executing integration
tests and unit tests at the same time allows more tests to fail than executing
integration tests alone.
NOTE: The return exit code can be customized with the `EARTHACCESS_ALLOWABLE_FAILURE_STATUS_CODE`
environment variable.
"""
if exitstatus != pytest.ExitCode.TESTS_FAILED:
# Exit status 1 in PyTest indicates "Tests were collected and run but some of
Expand All @@ -30,9 +34,15 @@ def pytest_sessionfinish(session, exitstatus):
# want to defer to original pytest behavior.
return


failure_rate = (100.0 * session.testsfailed) / session.testscollected
if failure_rate <= ACCEPTABLE_FAILURE_RATE:
session.exitstatus = 99
warn(
'\nWARNING: The integration test suite has been allowed to pass because the '
'failure rate was less than a hardcoded threshold. For more details see:\n'
'tests/integration/conftest.py.'
)
session.exitstatus = os.environ.get('EARTHACCESS_ALLOWABLE_FAILURE_STATUS_CODE', 99)


@pytest.fixture
Expand Down

0 comments on commit c5ba49b

Please sign in to comment.