From f2e8d3799f5f4e41b6e8f78c5d5ab341a759b35b Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Tue, 29 Sep 2026 17:37:33 -0700 Subject: [PATCH 01/10] ci: pin numpy<2.5 for nightly-numba-cuda and add cccl for numba-cuda-mlir Two targeted fixes for the recurring failures tracked in #2748: - nightly-numba-cuda: pin numpy<2.5. numba-cuda 0.30.4 references np.row_stack, removed in NumPy 2.5, so every nightly numba-cuda job crashes at collection. Tracked upstream in NVIDIA/numba-cuda#907; drop the cap once a fixed wheel is on PyPI. - nightly-numba-cuda-mlir: add cccl to cuda-toolkit extras. NVRTC currently fails with "catastrophic error: cannot open source file 'nv/target'" when compiling cooperative_groups/details/info.h and curand_kernel.h; the cccl extra pulls nvidia-cuda-cccl, which drops the missing header at nvidia/cu13/include/nv/target. --- ci/tools/run-tests | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/tools/run-tests b/ci/tools/run-tests index cfa7e9d6a7a..df7b39a4b40 100755 --- a/ci/tools/run-tests +++ b/ci/tools/run-tests @@ -162,10 +162,12 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then elif [[ "${test_module}" == "nightly-numba-cuda" ]]; then echo "Installing pathfinder + bindings + core + test deps + numba-cuda" # numba-cuda's test-cuXX group deps (can't use --group for a wheel install): + # numpy<2.5: numba-cuda 0.30.4 references np.row_stack, removed in numpy 2.5. See NVIDIA/numba-cuda#907. PIP_ARGS+=( "cuda-toolkit[curand,cublas]==${TEST_CUDA_MAJOR_MINOR}.*" "numba-cuda[cu${TEST_CUDA_MAJOR}]" "cupy-cuda${TEST_CUDA_MAJOR}x" + "numpy<2.5" psutil cffi pytest-xdist pytest-benchmark filecheck ml_dtypes statistics ) elif [[ "${test_module}" == "nightly-numba-cuda-mlir" ]]; then @@ -173,9 +175,10 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then # curand/cublas + cupy: some numba-cuda-mlir tests (e.g. # test_fortran_contiguous) call cp.random, which dlopens libcurand at # runtime. Mirror the numba-cuda branch, which installs the same libs. + # cccl: NVRTC needs for cooperative_groups / curand_kernel.h. PIP_ARGS+=( "numba-cuda-mlir[cu${TEST_CUDA_MAJOR}]" - "cuda-toolkit[curand,cublas]==${TEST_CUDA_MAJOR_MINOR}.*" + "cuda-toolkit[curand,cublas,cccl]==${TEST_CUDA_MAJOR_MINOR}.*" "cupy-cuda${TEST_CUDA_MAJOR}x" ) fi From 43639501d3a6c58ebdfd0e13b7debcaa1f67afa0 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Tue, 29 Sep 2026 17:45:15 -0700 Subject: [PATCH 02/10] TEMP: add push trigger to ci-nightly.yml to verify fix on this PR Revert before merging. Same trick as https://github.com/NVIDIA/cuda-python/pull/2296/commits/8d51cf761dda86fc29eeb3e68aba8860e366ef19 --- .github/workflows/ci-nightly.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci-nightly.yml b/.github/workflows/ci-nightly.yml index a3179c1e155..de7d21dba53 100644 --- a/.github/workflows/ci-nightly.yml +++ b/.github/workflows/ci-nightly.yml @@ -17,6 +17,13 @@ concurrency: cancel-in-progress: true on: + # TEMP for #2748: fire the nightly workflow on PR pushes so this PR can verify + # the numba-cuda and numba-cuda-mlir fixes without waiting for the 2:17 AM cron. + # REVERT before merging. + push: + branches: + - "main" + - "pull-request/[0-9]+" schedule: # 2:17 AM UTC daily, after the midnight main CI build finishes. # Avoid minute 0 because GitHub documents high scheduled-workflow load From feb41f7c3a36da87e8fe470f4e3468ce8d8d5efa Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Tue, 29 Sep 2026 17:56:32 -0700 Subject: [PATCH 03/10] ci: deselect frozen-driver-table test on released cuda-core <=1.2.1 test_frozen_driver_table_covers_all_curesult_members was removed on main by #2792 (2026-09-09) but neither cuda-core-v1.2.0 nor v1.2.1 carry the revert. main cuda-bindings 13.4.1 adds three new CUresult members (MULTICAST_RESOURCE_FULL, INSUFFICIENT_LOADER_VERSION, FABRIC_NOT_READY) not present in the frozen table those releases ship, so the nightly-cuda-core job trips it on every run. Mirror the existing v1.0.1 NvlinkVersion pattern with a version-gated --deselect that drops automatically once the next cuda-core release ships the #2792 revert. --- .github/workflows/test-wheel-linux.yml | 13 +++++++++++-- .github/workflows/test-wheel-windows.yml | 6 ++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 2f8e9a47a02..d0322f2347f 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -520,8 +520,8 @@ jobs: # patterns that pytest 9.1 rejects; the main-side fix (#2212) has # not yet shipped in a cuda-core release. pip install "pytest<9.1" - # Version-gated deselect: drops automatically when a newer - # cuda-core release with the wrapper-mapping update ships. + # Version-gated deselects: drop automatically when a newer + # cuda-core release ships that removes each source. DESELECTS=() if python -c "from packaging.version import Version; import sys; sys.exit(0 if Version('${CUDA_CORE_RELEASED_VER}') <= Version('1.0.1') else 1)"; then # NvlinkVersion: v1.0.1's wrapper mapping predates @@ -531,6 +531,15 @@ jobs: --deselect 'tests/test_enum_coverage.py::test_wrapper_covers_all_binding_members[NvlinkVersion]' ) fi + if python -c "from packaging.version import Version; import sys; sys.exit(0 if Version('${CUDA_CORE_RELEASED_VER}') <= Version('1.2.1') else 1)"; then + # test_frozen_driver_table_covers_all_curesult_members: v1.2.1's + # frozen table (capped at CTK 13.1.1) predates 3 CUresult members + # added by main cuda-bindings 13.4.1. Reverted on main by #2792; + # drop this deselect once a cuda-core release with the revert ships. + DESELECTS+=( + --deselect 'tests/test_utils_enum_explanations_helpers.py::test_frozen_driver_table_covers_all_curesult_members' + ) + fi pytest -rxXs -v --randomly-dont-reorganize \ "${DESELECTS[@]}" \ tests/ diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 1b9fc0ff9cb..bfd3c2f6bd6 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -517,6 +517,12 @@ jobs: --deselect 'tests/test_memory.py::test_non_managed_resources_report_not_managed[pinned]' ) fi + if python -c "from packaging.version import Version; import sys; sys.exit(0 if Version('${CUDA_CORE_RELEASED_VER}') <= Version('1.2.1') else 1)"; then + # See linux step for rationale (#2792 revert). + DESELECTS+=( + --deselect 'tests/test_utils_enum_explanations_helpers.py::test_frozen_driver_table_covers_all_curesult_members' + ) + fi pytest -rxXs -v --randomly-dont-reorganize \ "${DESELECTS[@]}" \ tests/ From f106b1f02a3f31aed812cfa8552a3bb4aa0908ad Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Tue, 29 Sep 2026 20:03:27 -0700 Subject: [PATCH 04/10] ci: skip numba-cuda IPC tests and mlir test_device_record_copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nightly-numba-cuda: switch from `python -m numba.runtests` to `pytest` (numba-cuda's own CI uses pytest; PR #1987 landed the runtests form as a bring-up fix) so we can skip individual tests. Two workarounds: - pin pytest<9 (matches upstream's numba-cuda cap; NVIDIA/numba-cuda#637 covers the subTest breakage on 9). - --ignore-glob "**/test_ipc.py" — numba-cuda v0.30.4 accesses CUipcMemHandle.reserved, which cuda-bindings intentionally removed; numba-cuda is EOL upstream so no fix is coming (#2748). nightly-numba-cuda-mlir: add --deselect for TestCudaDeviceRecordWithRecord.test_device_record_copy — the WithRecord fixture uses np.recarray (uninitialized memory), so when the float32 field encodes a NaN, `np.testing.assert_equal` trips on NaN != NaN even though the copy round-trip is correct. Filed upstream as NVIDIA/numba-cuda-mlir#341. --- .github/workflows/test-wheel-linux.yml | 13 ++++++++++++- .github/workflows/test-wheel-windows.yml | 7 ++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index d0322f2347f..251051ed662 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -469,7 +469,14 @@ jobs: - name: Run numba-cuda tests if: ${{ inputs.test-mode == 'nightly-numba-cuda' }} - run: python -m numba.runtests numba.cuda.tests + run: | + # pytest<9: numba-cuda's subTest-based tests break under pytest 9's + # xdist subtest handling. See NVIDIA/numba-cuda#637. + pip install "pytest<9" + # test_ipc: numba-cuda v0.30.4 accesses CUipcMemHandle.reserved, which + # cuda-bindings intentionally removed. numba-cuda is EOL upstream, so + # no fix is coming. See cuda-python#2748. + pytest -v --pyargs numba.cuda.tests --ignore-glob="**/test_ipc.py" - name: Checkout numba-cuda-mlir tests at matching tag if: ${{ inputs.test-mode == 'nightly-numba-cuda-mlir' && env.NUMBA_CUDA_MLIR_VER != '' }} @@ -495,9 +502,13 @@ jobs: # Skip tests/benchmarks/ and tests/doc_examples/ — they import the # numba package at collection time, which cuSIMT intentionally does # not depend on. See NVIDIA/numba-cuda-mlir#136. + # test_device_record_copy (WithRecord variant): flaky NaN vs NaN + # compare from uninitialized np.recarray memory. See + # NVIDIA/numba-cuda-mlir#341. pytest -rxXs -v \ --ignore=tests/benchmarks \ --ignore=tests/doc_examples \ + --deselect 'tests/numba_cuda_tests/cudadrv/test_cuda_devicerecord.py::TestCudaDeviceRecordWithRecord::test_device_record_copy' \ tests/ popd diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index bfd3c2f6bd6..93199fd753f 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -453,7 +453,10 @@ jobs: - name: Run numba-cuda tests if: ${{ inputs.test-mode == 'nightly-numba-cuda' }} shell: bash --noprofile --norc -xeuo pipefail {0} - run: python -m numba.runtests numba.cuda.tests + run: | + # See linux step for rationale (pytest<9 and test_ipc skip). + pip install "pytest<9" + pytest -v --pyargs numba.cuda.tests --ignore-glob="**/test_ipc.py" - name: Checkout numba-cuda-mlir tests at matching tag if: ${{ inputs.test-mode == 'nightly-numba-cuda-mlir' && env.NUMBA_CUDA_MLIR_VER != '' }} @@ -476,9 +479,11 @@ jobs: pushd numba-cuda-mlir-released pip install --upgrade "pip>=25.1" pip install --group test + # See linux step for rationale on the --deselect. pytest -rxXs -v \ --ignore=tests/benchmarks \ --ignore=tests/doc_examples \ + --deselect 'tests/numba_cuda_tests/cudadrv/test_cuda_devicerecord.py::TestCudaDeviceRecordWithRecord::test_device_record_copy' \ tests/ popd From f0682bf602a1a2d4b0fea929bfdfd9fdbf8416b1 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Tue, 29 Sep 2026 20:09:09 -0700 Subject: [PATCH 05/10] ci: move pytest<9 pin for numba-cuda into ci/tools/run-tests Keeps pip constraints in the install step alongside numpy<2.5, instead of a separate `pip install` in the workflow. --- .github/workflows/test-wheel-linux.yml | 3 --- .github/workflows/test-wheel-windows.yml | 3 +-- ci/tools/run-tests | 2 ++ 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 251051ed662..80c61c089db 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -470,9 +470,6 @@ jobs: - name: Run numba-cuda tests if: ${{ inputs.test-mode == 'nightly-numba-cuda' }} run: | - # pytest<9: numba-cuda's subTest-based tests break under pytest 9's - # xdist subtest handling. See NVIDIA/numba-cuda#637. - pip install "pytest<9" # test_ipc: numba-cuda v0.30.4 accesses CUipcMemHandle.reserved, which # cuda-bindings intentionally removed. numba-cuda is EOL upstream, so # no fix is coming. See cuda-python#2748. diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 93199fd753f..89dda98bb19 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -454,8 +454,7 @@ jobs: if: ${{ inputs.test-mode == 'nightly-numba-cuda' }} shell: bash --noprofile --norc -xeuo pipefail {0} run: | - # See linux step for rationale (pytest<9 and test_ipc skip). - pip install "pytest<9" + # See linux step for rationale on the test_ipc skip. pytest -v --pyargs numba.cuda.tests --ignore-glob="**/test_ipc.py" - name: Checkout numba-cuda-mlir tests at matching tag diff --git a/ci/tools/run-tests b/ci/tools/run-tests index df7b39a4b40..573d45862c5 100755 --- a/ci/tools/run-tests +++ b/ci/tools/run-tests @@ -163,11 +163,13 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then echo "Installing pathfinder + bindings + core + test deps + numba-cuda" # numba-cuda's test-cuXX group deps (can't use --group for a wheel install): # numpy<2.5: numba-cuda 0.30.4 references np.row_stack, removed in numpy 2.5. See NVIDIA/numba-cuda#907. + # pytest<9: numba-cuda's subTest usage breaks under pytest 9 + xdist. See NVIDIA/numba-cuda#637. PIP_ARGS+=( "cuda-toolkit[curand,cublas]==${TEST_CUDA_MAJOR_MINOR}.*" "numba-cuda[cu${TEST_CUDA_MAJOR}]" "cupy-cuda${TEST_CUDA_MAJOR}x" "numpy<2.5" + "pytest<9" psutil cffi pytest-xdist pytest-benchmark filecheck ml_dtypes statistics ) elif [[ "${test_module}" == "nightly-numba-cuda-mlir" ]]; then From 31f7af780618d874c8cfd2b6507f222a2d2a702c Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Tue, 29 Sep 2026 20:23:11 -0700 Subject: [PATCH 06/10] ci: downgrade pytest to <9 as a second pip call for numba-cuda cuda_core's test-cuXX group hard-pins pytest==9.1.0, so pip can't satisfy that and pytest<9 in one solve. Split the numba-cuda pytest pin into a second `pip install "pytest<9"` after the main install. --- ci/tools/run-tests | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ci/tools/run-tests b/ci/tools/run-tests index 573d45862c5..8b7297b71db 100755 --- a/ci/tools/run-tests +++ b/ci/tools/run-tests @@ -163,13 +163,11 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then echo "Installing pathfinder + bindings + core + test deps + numba-cuda" # numba-cuda's test-cuXX group deps (can't use --group for a wheel install): # numpy<2.5: numba-cuda 0.30.4 references np.row_stack, removed in numpy 2.5. See NVIDIA/numba-cuda#907. - # pytest<9: numba-cuda's subTest usage breaks under pytest 9 + xdist. See NVIDIA/numba-cuda#637. PIP_ARGS+=( "cuda-toolkit[curand,cublas]==${TEST_CUDA_MAJOR_MINOR}.*" "numba-cuda[cu${TEST_CUDA_MAJOR}]" "cupy-cuda${TEST_CUDA_MAJOR}x" "numpy<2.5" - "pytest<9" psutil cffi pytest-xdist pytest-benchmark filecheck ml_dtypes statistics ) elif [[ "${test_module}" == "nightly-numba-cuda-mlir" ]]; then @@ -186,6 +184,15 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then fi pip install "${PIP_ARGS[@]}" + + if [[ "${test_module}" == "nightly-numba-cuda" ]]; then + # Downgrade pytest for numba-cuda: cuda_core's test-cuXX group hard-pins + # pytest==9.1.0, but numba-cuda's subTest usage breaks under pytest 9. + # Can't add pytest<9 to PIP_ARGS above because pip can't satisfy both + # ==9.1.0 and <9 in one solve. See NVIDIA/numba-cuda#637. + pip install "pytest<9" + fi + echo "Nightly install complete — installed packages:" pip list popd From 13a2035d61fd35c5f9ff63b04b91d459668a0499 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Tue, 29 Sep 2026 20:43:37 -0700 Subject: [PATCH 07/10] ci: use --import-mode=importlib for numba-cuda pytest run Default `prepend` import mode walks up until it finds a directory without `__init__.py`. numba-cuda's on-disk layout is `site-packages/numba_cuda/numba/cuda/tests/...`, and `numba/` here is a namespace subdir with no `__init__.py`, so pytest treats it as the rootpath and imports each test file as `cuda.tests.<...>`. That collides with cuda-bindings' top-level `cuda` package (which has no `tests` subpackage), giving `ModuleNotFoundError: No module named 'cuda.tests'` at collection. `--import-mode=importlib` sidesteps the sys.path walk and imports each test module by its correct dotted path. --- .github/workflows/test-wheel-linux.yml | 7 ++++++- .github/workflows/test-wheel-windows.yml | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 80c61c089db..e8ae452a124 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -470,10 +470,15 @@ jobs: - name: Run numba-cuda tests if: ${{ inputs.test-mode == 'nightly-numba-cuda' }} run: | + # --import-mode=importlib: numba-cuda's on-disk layout is + # site-packages/numba_cuda/numba/cuda/tests/... where numba/ is a + # namespace subdir (no __init__.py). Default prepend import mode + # stops there and imports each test file as `cuda.tests.<...>`, + # which collides with cuda-bindings' top-level cuda package. # test_ipc: numba-cuda v0.30.4 accesses CUipcMemHandle.reserved, which # cuda-bindings intentionally removed. numba-cuda is EOL upstream, so # no fix is coming. See cuda-python#2748. - pytest -v --pyargs numba.cuda.tests --ignore-glob="**/test_ipc.py" + pytest -v --import-mode=importlib --pyargs numba.cuda.tests --ignore-glob="**/test_ipc.py" - name: Checkout numba-cuda-mlir tests at matching tag if: ${{ inputs.test-mode == 'nightly-numba-cuda-mlir' && env.NUMBA_CUDA_MLIR_VER != '' }} diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 89dda98bb19..b139f7ea5a9 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -454,8 +454,8 @@ jobs: if: ${{ inputs.test-mode == 'nightly-numba-cuda' }} shell: bash --noprofile --norc -xeuo pipefail {0} run: | - # See linux step for rationale on the test_ipc skip. - pytest -v --pyargs numba.cuda.tests --ignore-glob="**/test_ipc.py" + # See linux step for rationale on --import-mode and the test_ipc skip. + pytest -v --import-mode=importlib --pyargs numba.cuda.tests --ignore-glob="**/test_ipc.py" - name: Checkout numba-cuda-mlir tests at matching tag if: ${{ inputs.test-mode == 'nightly-numba-cuda-mlir' && env.NUMBA_CUDA_MLIR_VER != '' }} From 3bc4accbf59569f420c860eab1eee5f01c666244 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Tue, 29 Sep 2026 20:57:21 -0700 Subject: [PATCH 08/10] ci: run numba-cuda tests from checked-out source, mirroring mlir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same shape as the numba-cuda-mlir step. run-tests exposes NUMBA_CUDA_VER; the workflow checks out NVIDIA/numba-cuda at the matching tag and runs pytest from numba-cuda-released/testing/, so upstream's testing/pytest.ini kicks in (consider_namespace_packages=true + --pyargs numba.cuda.tests) and we stop caring about the site-packages namespace-package layout. Also mirror the wheel-only gate: in wheels mode, set NUMBA_CUDA_TEST_WHEEL_ONLY=1 so numba-cuda skips tests that need CTK executables. Binary-generated tests self-skip via `@unittest.skipIf(not NUMBA_CUDA_TEST_BIN_DIR, ...)`, so no `make -j` step is needed — mirrors mlir, which has no Makefile. Drops the --import-mode=importlib workaround from the prior commit; pytest.ini's consider_namespace_packages does the job. --- .github/workflows/test-wheel-linux.yml | 27 ++++++++++++++++++------ .github/workflows/test-wheel-windows.yml | 21 +++++++++++++++--- ci/tools/run-tests | 9 ++++++++ 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index e8ae452a124..db157f824ba 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -467,18 +467,31 @@ jobs: pytest -rxXs -v tests/test_utils.py tests/example_tests/ popd + - name: Checkout numba-cuda tests at matching tag + if: ${{ inputs.test-mode == 'nightly-numba-cuda' && env.NUMBA_CUDA_VER != '' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: NVIDIA/numba-cuda + ref: v${{ env.NUMBA_CUDA_VER }} + path: numba-cuda-released + - name: Run numba-cuda tests - if: ${{ inputs.test-mode == 'nightly-numba-cuda' }} + if: ${{ inputs.test-mode == 'nightly-numba-cuda' && env.NUMBA_CUDA_VER != '' }} + env: + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} run: | - # --import-mode=importlib: numba-cuda's on-disk layout is - # site-packages/numba_cuda/numba/cuda/tests/... where numba/ is a - # namespace subdir (no __init__.py). Default prepend import mode - # stops there and imports each test file as `cuda.tests.<...>`, - # which collides with cuda-bindings' top-level cuda package. + if [[ "${LOCAL_CTK}" != 1 ]]; then + # Let numba-cuda skip inputs that require toolkit executables. + export NUMBA_CUDA_TEST_WHEEL_ONLY=1 + fi + # Run from numba-cuda's testing/ dir so its pytest.ini kicks in + # (consider_namespace_packages=true + --pyargs numba.cuda.tests). # test_ipc: numba-cuda v0.30.4 accesses CUipcMemHandle.reserved, which # cuda-bindings intentionally removed. numba-cuda is EOL upstream, so # no fix is coming. See cuda-python#2748. - pytest -v --import-mode=importlib --pyargs numba.cuda.tests --ignore-glob="**/test_ipc.py" + pushd numba-cuda-released/testing + pytest -v --ignore-glob="**/test_ipc.py" + popd - name: Checkout numba-cuda-mlir tests at matching tag if: ${{ inputs.test-mode == 'nightly-numba-cuda-mlir' && env.NUMBA_CUDA_MLIR_VER != '' }} diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index b139f7ea5a9..3ed73ca8b0d 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -450,12 +450,27 @@ jobs: pytest -rxXs -v tests/test_utils.py tests/example_tests/ popd + - name: Checkout numba-cuda tests at matching tag + if: ${{ inputs.test-mode == 'nightly-numba-cuda' && env.NUMBA_CUDA_VER != '' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: NVIDIA/numba-cuda + ref: v${{ env.NUMBA_CUDA_VER }} + path: numba-cuda-released + - name: Run numba-cuda tests - if: ${{ inputs.test-mode == 'nightly-numba-cuda' }} + if: ${{ inputs.test-mode == 'nightly-numba-cuda' && env.NUMBA_CUDA_VER != '' }} + env: + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} shell: bash --noprofile --norc -xeuo pipefail {0} run: | - # See linux step for rationale on --import-mode and the test_ipc skip. - pytest -v --import-mode=importlib --pyargs numba.cuda.tests --ignore-glob="**/test_ipc.py" + if [[ "${LOCAL_CTK}" != 1 ]]; then + export NUMBA_CUDA_TEST_WHEEL_ONLY=1 + fi + # See linux step for rationale on running from testing/ and the test_ipc skip. + pushd numba-cuda-released/testing + pytest -v --ignore-glob="**/test_ipc.py" + popd - name: Checkout numba-cuda-mlir tests at matching tag if: ${{ inputs.test-mode == 'nightly-numba-cuda-mlir' && env.NUMBA_CUDA_MLIR_VER != '' }} diff --git a/ci/tools/run-tests b/ci/tools/run-tests index 8b7297b71db..c882fae0517 100755 --- a/ci/tools/run-tests +++ b/ci/tools/run-tests @@ -205,6 +205,15 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then if [[ -n "${GITHUB_ENV:-}" ]]; then echo "NUMBA_CUDA_MLIR_VER=${installed_ver}" >> "${GITHUB_ENV}" fi + elif [[ "${test_module}" == "nightly-numba-cuda" ]]; then + # Expose the installed numba-cuda version so the workflow can + # actions/checkout the matching v tag from NVIDIA/numba-cuda + # (the wheel does not ship test_*.py files at a pytest-discoverable + # top level; upstream's testing/pytest.ini is required to run them). + installed_ver=$(pip show numba-cuda | awk '/^Version:/{print $2}') + if [[ -n "${GITHUB_ENV:-}" ]]; then + echo "NUMBA_CUDA_VER=${installed_ver}" >> "${GITHUB_ENV}" + fi fi fi fi From b53d482b08abe24b0e2acd0fa92e93f6decf6ff0 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Tue, 29 Sep 2026 21:55:55 -0700 Subject: [PATCH 09/10] ci: swap --ignore-glob for -k 'not TestIpc' in numba-cuda pytest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pytest's --ignore-glob="**/test_ipc.py" didn't match through the --pyargs-resolved site-packages path (5 tests still ran and failed with the CUipcMemHandle.reserved AttributeError on linux-64). Match by class name instead — the affected tests live in TestIpcMemory and TestIpcStaged, both starting with "TestIpc", so this is unambiguous. --- .github/workflows/test-wheel-linux.yml | 9 +++++---- .github/workflows/test-wheel-windows.yml | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index db157f824ba..0bb470b26c3 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -486,11 +486,12 @@ jobs: fi # Run from numba-cuda's testing/ dir so its pytest.ini kicks in # (consider_namespace_packages=true + --pyargs numba.cuda.tests). - # test_ipc: numba-cuda v0.30.4 accesses CUipcMemHandle.reserved, which - # cuda-bindings intentionally removed. numba-cuda is EOL upstream, so - # no fix is coming. See cuda-python#2748. + # -k "not TestIpc": numba-cuda v0.30.4 accesses CUipcMemHandle.reserved, + # which cuda-bindings intentionally removed. numba-cuda is EOL upstream, + # so no fix is coming. See cuda-python#2748. The affected tests all + # live in the two TestIpcMemory/TestIpcStaged classes in test_ipc.py. pushd numba-cuda-released/testing - pytest -v --ignore-glob="**/test_ipc.py" + pytest -v -k "not TestIpc" popd - name: Checkout numba-cuda-mlir tests at matching tag diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 3ed73ca8b0d..7abfc74126b 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -467,9 +467,9 @@ jobs: if [[ "${LOCAL_CTK}" != 1 ]]; then export NUMBA_CUDA_TEST_WHEEL_ONLY=1 fi - # See linux step for rationale on running from testing/ and the test_ipc skip. + # See linux step for rationale on running from testing/ and -k. pushd numba-cuda-released/testing - pytest -v --ignore-glob="**/test_ipc.py" + pytest -v -k "not TestIpc" popd - name: Checkout numba-cuda-mlir tests at matching tag From 4866459342e4a7fbfa931a31f7caf550c534e732 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Wed, 30 Sep 2026 07:27:33 -0700 Subject: [PATCH 10/10] Revert "TEMP: add push trigger to ci-nightly.yml to verify fix on this PR" This reverts commit 43639501d3a6c58ebdfd0e13b7debcaa1f67afa0. --- .github/workflows/ci-nightly.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci-nightly.yml b/.github/workflows/ci-nightly.yml index de7d21dba53..a3179c1e155 100644 --- a/.github/workflows/ci-nightly.yml +++ b/.github/workflows/ci-nightly.yml @@ -17,13 +17,6 @@ concurrency: cancel-in-progress: true on: - # TEMP for #2748: fire the nightly workflow on PR pushes so this PR can verify - # the numba-cuda and numba-cuda-mlir fixes without waiting for the 2:17 AM cron. - # REVERT before merging. - push: - branches: - - "main" - - "pull-request/[0-9]+" schedule: # 2:17 AM UTC daily, after the midnight main CI build finishes. # Avoid minute 0 because GitHub documents high scheduled-workflow load