Build conda python packages independently of the Python version

Build one set of conda python packages per CUDA x arch instead of one
per Python version, taking conda-python-build from 16 jobs to 4.

All three python packages have to become version-independent for the
matrix filter to be safe, since they share a single output channel:

- cuopt: abi3 via 'version_independent: true', built against the
  minimum supported Python. 'python-gil' is dropped because
  'python-abi3' already depends on it transitively, and cython is
  raised to >=3.2.2 for its Limited API support.
- cuopt-server, cuopt-sh-client: 'noarch: python'. Neither builds
  extension modules.

build.sh passes the abi3 floor only for cuopt, so the two pure packages
keep using the plain install args.

Depends on #1731, which sets 'wheel.py-api' in python/cuopt.

Signed-off-by: Ramakrishna Prabhu <ramakrishnap@nvidia.com>

Author: ramakrishnap-nv

.github/workflows/build.yaml

@@ -81,6 +81,8 @@ jobs:
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
       script: ci/build_python.sh
+      # Build a conda package for each CUDA x ARCH x minimum supported Python version
+      matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber)))
   upload-conda:
     needs: [cpp-build, python-build]
     permissions:

.github/workflows/pr.yaml

@@ -412,6 +412,8 @@ jobs:
       build_type: pull-request
       build-datetime: ${{ needs.build-details.outputs.build-datetime }}
       script: ci/build_python.sh
+      # Build a conda package for each CUDA x ARCH x minimum supported Python version
+      matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber)))
   conda-python-tests:
     needs: [conda-python-build, changed-files]
     permissions:

build.sh

@@ -432,8 +432,15 @@ fi
 if buildAll || hasArg cuopt; then
     cd "${REPODIR}"/python/cuopt
 
+    # Only 'cuopt' builds extension modules, so the stable ABI floor applies to it
+    # alone. If 'RAPIDS_PY_VERSION' is set, use it as that floor.
+    CUOPT_PYTHON_ARGS_FOR_INSTALL=("${PYTHON_ARGS_FOR_INSTALL[@]}")
+    if [ -n "${RAPIDS_PY_VERSION:-}" ]; then
+        CUOPT_PYTHON_ARGS_FOR_INSTALL+=(--config-settings "skbuild.wheel.py-api=cp${RAPIDS_PY_VERSION//./}")
+    fi
+
     SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${LIBCUOPT_BUILD_DIR};-DCMAKE_CUDA_ARCHITECTURES=${CUOPT_CMAKE_CUDA_ARCHITECTURES};$(IFS=';'; echo "${EXTRA_CMAKE_ARGS[*]}")" \
-        python "${PYTHON_ARGS_FOR_INSTALL[@]}" .
+        python "${CUOPT_PYTHON_ARGS_FOR_INSTALL[@]}" .
 fi
 
 # Build and install the cuopt_server Python package

ci/build_docs.sh

@@ -18,7 +18,7 @@ export RAPIDS_VERSION_MAJOR_MINOR
 
 rapids-logger "Downloading artifacts from previous jobs"
 CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcuopt cuopt --cuda "$RAPIDS_CUDA_VERSION")")
-PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cuopt cuopt --py "$RAPIDS_PY_VERSION" --cuda "$RAPIDS_CUDA_VERSION")")
+PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cuopt cuopt --stable --cuda "$RAPIDS_CUDA_VERSION")")
 
 rapids-logger "Generating conda environment YAML"
 

ci/build_python.sh

@@ -67,5 +67,5 @@ rattler-build build --recipe conda/recipes/cuopt-sh-client \
 # tracked in https://github.com/prefix-dev/rattler-build/issues/1424
 rm -rf "$RAPIDS_CONDA_BLD_OUTPUT_DIR"/build_cache
 
-RAPIDS_PACKAGE_NAME="$(rapids-artifact-name conda_python cuopt cuopt --py "$RAPIDS_PY_VERSION" --cuda "$RAPIDS_CUDA_VERSION")"
+RAPIDS_PACKAGE_NAME="$(rapids-artifact-name conda_python cuopt cuopt --stable --cuda "$RAPIDS_CUDA_VERSION")"
 export RAPIDS_PACKAGE_NAME

ci/test_notebooks.sh

@@ -12,7 +12,7 @@ conda config --set channel_priority strict
 
 rapids-logger "Downloading artifacts from previous jobs"
 CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcuopt cuopt --cuda "$RAPIDS_CUDA_VERSION")")
-PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cuopt cuopt --py "$RAPIDS_PY_VERSION" --cuda "$RAPIDS_CUDA_VERSION")")
+PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cuopt cuopt --stable --cuda "$RAPIDS_CUDA_VERSION")")
 
 rapids-logger "Generate notebook testing dependencies"
 

ci/test_python.sh

@@ -12,7 +12,7 @@ conda config --set channel_priority strict
 
 rapids-logger "Downloading artifacts from previous jobs"
 CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcuopt cuopt --cuda "$RAPIDS_CUDA_VERSION")")
-PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cuopt cuopt --py "$RAPIDS_PY_VERSION" --cuda "$RAPIDS_CUDA_VERSION")")
+PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cuopt cuopt --stable --cuda "$RAPIDS_CUDA_VERSION")")
 
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \

conda/recipes/cuopt-server/recipe.yaml

@@ -5,8 +5,6 @@ schema_version: 1
 context:
   version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }}
   datetime_string: '${{ env.get("RAPIDS_DATETIME_STRING") }}'
-  py_version: ${{ env.get("RAPIDS_PY_VERSION") }}
-  py_buildstring: ${{ py_version | version_to_buildstring }}
   head_rev: '${{ git.head_rev(".")[:8] }}'
 
 package:
@@ -17,7 +15,8 @@ source:
   path: ../../..
 
 build:
-  string: py${{ py_buildstring }}_${{ datetime_string }}_${{ head_rev }}
+  noarch: python
+  string: ${{ datetime_string }}_${{ head_rev }}
   script:
     content: |
       # We pass an explicit matrix-entry here along with `disable-cuda=true` to
@@ -28,7 +27,7 @@ build:
 requirements:
   host:
     - pip
-    - python-gil =${{ py_version }}
+    - python >=3.11
     - rapids-build-backend >=0.4.0,<0.5.0
     - setuptools>=77.0.0
   run:

conda/recipes/cuopt-sh-client/recipe.yaml

@@ -6,8 +6,6 @@ context:
   version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }}
   minor_version: ${{ (version | split("."))[:2] | join(".") }}
   datetime_string: '${{ env.get("RAPIDS_DATETIME_STRING") }}'
-  py_version: ${{ env.get("RAPIDS_PY_VERSION") }}
-  py_buildstring: ${{ py_version | version_to_buildstring }}
   head_rev: '${{ git.head_rev(".")[:8] }}'
 
 package:
@@ -18,15 +16,16 @@ source:
   path: ../../..
 
 build:
-  string: py${{ py_buildstring }}_${{ datetime_string }}_${{ head_rev }}
+  noarch: python
+  string: ${{ datetime_string }}_${{ head_rev }}
   script:
     content: |
       ./build.sh cuopt_sh_client
 
 requirements:
   host:
     - pip
-    - python-gil =${{ py_version }}
+    - python >=3.11
     - setuptools>=77.0.0
   run:
     - msgpack-python =1.2.1

conda/recipes/cuopt/recipe.yaml

@@ -8,8 +8,9 @@ context:
   cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }}
   cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}'
   datetime_string: '${{ env.get("RAPIDS_DATETIME_STRING") }}'
-  py_version: ${{ env.get("RAPIDS_PY_VERSION") }}
-  py_buildstring: ${{ py_version | version_to_buildstring }}
+  py_abi_min: ${{ env.get("RAPIDS_PY_VERSION") }}
+  py_buildstring: ${{ py_abi_min | version_to_buildstring }}
+  py_runtime_latest: "3.14"
   head_rev: '${{ git.head_rev(".")[:8] }}'
 
 package:
@@ -20,7 +21,9 @@ source:
   path: ../../..
 
 build:
-  string: cuda${{ cuda_major }}_py${{ py_buildstring }}_${{ datetime_string }}_${{ head_rev }}
+  python:
+    version_independent: true
+  string: cuda${{ cuda_major }}_cp${{ py_buildstring }}_abi3_${{ datetime_string }}_${{ head_rev }}
   dynamic_linking:
     overlinking_behavior: error
   prefix_detection:
@@ -42,6 +45,7 @@ build:
       CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }}
       NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }}
       PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }}
+      RAPIDS_PY_VERSION: ${{ py_abi_min }}
       SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }}
       SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }}
       SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }}
@@ -71,11 +75,12 @@ requirements:
     - ${{ stdlib("c") }}
   host:
     - cuda-version =${{ cuda_version }}
-    - cython >=3.0.0
+    - cython >=3.2.2
     - libcuopt =${{ version }}
     - pip
     - pylibraft =${{ minor_version }}
-    - python-gil =${{ py_version }}
+    - python =${{ py_abi_min }}
+    - python-abi3 ${{ py_abi_min }}.*
     - rapids-build-backend >=0.4.0,<0.5.0
     - rmm =${{ minor_version }}
     - scikit-build-core>=0.11.0
@@ -105,6 +110,7 @@ tests:
   - python:
       imports:
         - cuopt
+      python_version: ${{ py_runtime_latest }}.*
       pip_check: false
 
 about: