From bb30a22f5cf08e7a1da5f4836dee1e974fbf027c Mon Sep 17 00:00:00 2001 From: James Lamb <jlamb@nvidia.com> Date: Mon, 9 Sep 2024 07:14:20 -0700 Subject: [PATCH] Add support for Python 3.12 (#276) Contributes to https://github.com/rapidsai/build-planning/issues/40 This PR adds support for Python 3.12. ## Notes for Reviewers This is part of ongoing work to add Python 3.12 support across RAPIDS. It temporarily introduces a build/test matrix including Python 3.12, from https://github.com/rapidsai/shared-workflows/pull/213. A follow-up PR will revert back to pointing at the `branch-24.10` branch of `shared-workflows` once all RAPIDS repos have added Python 3.12 support. ### This will fail until all dependencies have been updates to Python 3.12 CI here is expected to fail until all of this project's upstream dependencies support Python 3.12. This can be merged whenever all CI jobs are passing. Authors: - James Lamb (https://github.com/jameslamb) - Peter Andreas Entschev (https://github.com/pentschev) Approvers: - Bradley Dice (https://github.com/bdice) - Peter Andreas Entschev (https://github.com/pentschev) URL: https://github.com/rapidsai/ucxx/pull/276 --- .github/workflows/build.yaml | 18 +++++----- .github/workflows/pr.yaml | 24 ++++++------- .github/workflows/test.yaml | 10 +++--- .../all_cuda-118_arch-x86_64.yaml | 2 +- .../all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/ucxx/conda_build_config.yaml | 1 + cpp/python/src/future.cpp | 34 ++----------------- dependencies.yaml | 6 +++- python/distributed-ucxx/pyproject.toml | 1 + 9 files changed, 37 insertions(+), 61 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5fdfcff0..afe76e3b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: conda-cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -38,7 +38,7 @@ jobs: if: github.ref_type == 'branch' needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: upload-conda: needs: [conda-cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -59,7 +59,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-libucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: wheel-publish-libucxx: needs: wheel-build-libucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -80,7 +80,7 @@ jobs: wheel-build-ucxx: needs: wheel-build-libucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -90,7 +90,7 @@ jobs: wheel-publish-ucxx: needs: wheel-build-ucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -100,7 +100,7 @@ jobs: package-type: python wheel-build-distributed-ucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -110,7 +110,7 @@ jobs: wheel-publish-distributed-ucxx: needs: [wheel-build-ucxx, wheel-build-distributed-ucxx] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c57d575d..6d771cfd 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,22 +24,22 @@ jobs: - wheel-build-distributed-ucxx - wheel-tests-distributed-ucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@python-3.12 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@python-3.12 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 with: build_type: pull-request docs-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -49,21 +49,21 @@ jobs: conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-distributed-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 with: build_type: pull-request script: "ci/test_python_distributed.sh" @@ -71,21 +71,21 @@ jobs: wheel-build-libucxx: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: ci/build_wheel_libucxx.sh wheel-build-ucxx: needs: wheel-build-libucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: ci/build_wheel_ucxx.sh wheel-tests-ucxx: needs: wheel-build-ucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" @@ -93,14 +93,14 @@ jobs: wheel-build-distributed-ucxx: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: ci/build_wheel_distributed_ucxx.sh wheel-tests-distributed-ucxx: needs: [wheel-build-ucxx, wheel-build-distributed-ucxx] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 467a8d1d..ea8b738b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -25,7 +25,7 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-distributed-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 with: build_type: nightly script: "ci/test_python_distributed.sh" @@ -44,7 +44,7 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" wheel-tests-ucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -54,7 +54,7 @@ jobs: script: ci/test_wheel_ucxx.sh wheel-tests-distributed-ucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 2f285d2f..135a01be 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -34,7 +34,7 @@ dependencies: - pytest-asyncio - pytest-rerunfailures - pytest==7.* -- python>=3.10,<3.12 +- python>=3.10,<3.13 - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==24.10.*,>=0.0.0a0 - rmm==24.10.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index cf9815cc..b63f81c3 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -34,7 +34,7 @@ dependencies: - pytest-asyncio - pytest-rerunfailures - pytest==7.* -- python>=3.10,<3.12 +- python>=3.10,<3.13 - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==24.10.*,>=0.0.0a0 - rmm==24.10.*,>=0.0.0a0 diff --git a/conda/recipes/ucxx/conda_build_config.yaml b/conda/recipes/ucxx/conda_build_config.yaml index 36891b09..1fef0694 100644 --- a/conda/recipes/ucxx/conda_build_config.yaml +++ b/conda/recipes/ucxx/conda_build_config.yaml @@ -22,6 +22,7 @@ cmake: python: - 3.10 - 3.11 + - 3.12 ucx: - "==1.15.*" diff --git a/cpp/python/src/future.cpp b/cpp/python/src/future.cpp index b82daddf..0647e0e3 100644 --- a/cpp/python/src/future.cpp +++ b/cpp/python/src/future.cpp @@ -113,41 +113,13 @@ PyObject* create_python_future() return result; } -static PyCFunction get_future_method(const char* method_name) -{ - PyCFunction result = NULL; - - PyGILState_STATE state = PyGILState_Ensure(); - - PyObject* future_object = get_asyncio_future_object(); - if (PyErr_Occurred()) { - ucxx_trace_req("ucxx::python::%s, error getting asyncio.Future method object", __func__); - PyErr_Print(); - } - PyMethodDef* m = reinterpret_cast<PyTypeObject*>(future_object)->tp_methods; - - for (; m != NULL; ++m) { - if (m->ml_name && !strcmp(m->ml_name, method_name)) { - result = m->ml_meth; - break; - } - } - - if (!result) - PyErr_Format(PyExc_RuntimeError, "Unable to load function pointer for `Future.set_result`."); - - PyGILState_Release(state); - return result; -} - PyObject* future_set_result(PyObject* future, PyObject* value) { PyObject* result = NULL; PyGILState_STATE state = PyGILState_Ensure(); - PyCFunction f = get_future_method("set_result"); - result = f(future, value); + result = PyObject_CallMethodOneArg(future, set_result_str, value); if (PyErr_Occurred()) { ucxx_trace_req("ucxx::python::%s, error calling `set_result()` from `asyncio.Future` object", __func__); @@ -165,7 +137,6 @@ PyObject* future_set_exception(PyObject* future, PyObject* exception, const char PyObject* message_object = NULL; PyObject* message_tuple = NULL; PyObject* formed_exception = NULL; - PyCFunction f = NULL; PyGILState_STATE state = PyGILState_Ensure(); @@ -176,9 +147,8 @@ PyObject* future_set_exception(PyObject* future, PyObject* exception, const char formed_exception = PyObject_Call(exception, message_tuple, NULL); if (formed_exception == NULL) goto err; - f = get_future_method("set_exception"); + result = PyObject_CallMethodOneArg(future, set_exception_str, formed_exception); - result = f(future, formed_exception); goto finish; err: diff --git a/dependencies.yaml b/dependencies.yaml index 639e4112..329af493 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -272,8 +272,12 @@ dependencies: packages: - python=3.11 - matrix: + py: "3.12" packages: - - python>=3.10,<3.12 + - python=3.12 + - matrix: + packages: + - python>=3.10,<3.13 run_python_ucxx: common: - output_types: [conda, requirements, pyproject] diff --git a/python/distributed-ucxx/pyproject.toml b/python/distributed-ucxx/pyproject.toml index 78184b00..b1a80b09 100644 --- a/python/distributed-ucxx/pyproject.toml +++ b/python/distributed-ucxx/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] [project.entry-points."distributed.comm.backends"]