diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 67c451fbd6e..6942ef0009d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -108,3 +108,21 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: dask_cudf + trigger-pandas-tests: + if: inputs.build_type == 'nightly' + needs: wheel-build-cudf + runs-on: ubuntu-latest + steps: + - name: Checkout code repo + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - name: Trigger pandas-tests + env: + GH_TOKEN: ${{ github.token }} + run: | + gh workflow run pandas-tests.yaml \ + -f branch=${{ inputs.branch }} \ + -f sha=${{ inputs.sha }} \ + -f date=${{ inputs.date }} diff --git a/.github/workflows/pandas-tests.yaml b/.github/workflows/pandas-tests.yaml new file mode 100644 index 00000000000..60544294809 --- /dev/null +++ b/.github/workflows/pandas-tests.yaml @@ -0,0 +1,27 @@ +name: Pandas Test Job + +on: + workflow_dispatch: + inputs: + branch: + required: true + type: string + date: + required: true + type: string + sha: + required: true + type: string + +jobs: + pandas-tests: + # run the Pandas unit tests + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 + with: + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and .CUDA_VER == "12.2.2" )) + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + script: ci/cudf_pandas_scripts/pandas-tests/run.sh main diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 345ccbea45b..f84b1f42928 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -174,7 +174,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and .CUDA_VER == "12.2.2" )) build_type: pull-request script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr # Hide test failures because they exceed the GITHUB_STEP_SUMMARY output limit. diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 65aef37697e..c5ae2f3b5a8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -125,14 +125,3 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/cudf_pandas_scripts/run_tests.sh - pandas-tests: - # run the Pandas unit tests - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 - with: - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(min_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - script: ci/cudf_pandas_scripts/pandas-tests/run.sh main diff --git a/ci/cudf_pandas_scripts/pandas-tests/diff.sh b/ci/cudf_pandas_scripts/pandas-tests/diff.sh index cf80f383db4..f87a3a36fcc 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/diff.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/diff.sh @@ -8,14 +8,16 @@ # Hard-coded needs to match the version deduced by rapids-upload-artifacts-dir GH_JOB_NAME="pandas-tests-diff / build" +RAPIDS_FULL_VERSION=$(<./VERSION) rapids-logger "Github job name: ${GH_JOB_NAME}" +rapids-logger "Rapids version: ${RAPIDS_FULL_VERSION}" PY_VER="39" -MAIN_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.main-results.json -PR_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.pr-results.json +MAIN_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json +PR_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.pr-${RAPIDS_FULL_VERSION}-results.json rapids-logger "Fetching latest available results from nightly" -aws s3api list-objects-v2 --bucket rapids-downloads --prefix "nightly/" --query "sort_by(Contents[?ends_with(Key, '_py${PY_VER}.main-results.json')], &LastModified)[::-1].[Key]" --output text > s3_output.txt +aws s3api list-objects-v2 --bucket rapids-downloads --prefix "nightly/" --query "sort_by(Contents[?ends_with(Key, '_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json')], &LastModified)[::-1].[Key]" --output text > s3_output.txt read -r COMPARE_ENV < s3_output.txt export COMPARE_ENV diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index 1f70ca78c41..d13d31ad09f 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -6,8 +6,8 @@ set -euo pipefail PANDAS_TESTS_BRANCH=${1} - -rapids-logger "Running Pandas tests using $PANDAS_TESTS_BRANCH branch" +RAPIDS_FULL_VERSION=$(<./VERSION) +rapids-logger "Running Pandas tests using $PANDAS_TESTS_BRANCH branch and rapids-version $RAPIDS_FULL_VERSION" rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" @@ -27,9 +27,10 @@ bash python/cudf/cudf/pandas/scripts/run-pandas-tests.sh \ --dist worksteal \ --report-log=${PANDAS_TESTS_BRANCH}.json 2>&1 +SUMMARY_FILE_NAME=${PANDAS_TESTS_BRANCH}-24.06-results.json # summarize the results and save them to artifacts: -python python/cudf/cudf/pandas/scripts/summarize-test-results.py --output json pandas-testing/${PANDAS_TESTS_BRANCH}.json > pandas-testing/${PANDAS_TESTS_BRANCH}-results.json +python python/cudf/cudf/pandas/scripts/summarize-test-results.py --output json pandas-testing/${PANDAS_TESTS_BRANCH}.json > pandas-testing/${SUMMARY_FILE_NAME} RAPIDS_ARTIFACTS_DIR=${RAPIDS_ARTIFACTS_DIR:-"${PWD}/artifacts"} mkdir -p "${RAPIDS_ARTIFACTS_DIR}" -mv pandas-testing/${PANDAS_TESTS_BRANCH}-results.json ${RAPIDS_ARTIFACTS_DIR}/ -rapids-upload-to-s3 ${RAPIDS_ARTIFACTS_DIR}/${PANDAS_TESTS_BRANCH}-results.json "${RAPIDS_ARTIFACTS_DIR}" +mv pandas-testing/${SUMMARY_FILE_NAME} ${RAPIDS_ARTIFACTS_DIR}/ +rapids-upload-to-s3 ${RAPIDS_ARTIFACTS_DIR}/${SUMMARY_FILE_NAME} "${RAPIDS_ARTIFACTS_DIR}"