From 4568e9cdb6794c77cbcf88b1f89f7a5a81debb82 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 2 Jul 2026 15:01:42 +0000 Subject: [PATCH 1/9] ci/test-matrix.yml: re-enable arm64 gh200 nightly-standard row Reverts the disable in 4c70cfac05 now that the runner team has fixed the pool-side hang on stream-ordered memory allocator calls. --- ci/test-matrix.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/test-matrix.yml b/ci/test-matrix.yml index c9eafd4f521..54b49a405e3 100644 --- a/ci/test-matrix.yml +++ b/ci/test-matrix.yml @@ -97,9 +97,7 @@ linux: - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } } - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } } # nightly-standard (arm64 nightly-only runners — per runner team request) - # TODO: gh200 row disabled — currently hangs on stream-ordered memory - # allocator (cudaMallocAsync); runner pool needs fixing first. - # - { ARCH: 'arm64', PY_VER: '3.14', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'gh200', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-standard' } } + - { ARCH: 'arm64', PY_VER: '3.14', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'gh200', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-standard' } } - { ARCH: 'arm64', PY_VER: '3.14', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '2', DRIVER: 'latest', ENV: { MODE: 'nightly-standard' } } - { ARCH: 'arm64', PY_VER: '3.14t', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '2', DRIVER: 'latest', ENV: { MODE: 'nightly-standard' } } From 8d51cf761dda86fc29eeb3e68aba8860e366ef19 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 2 Jul 2026 15:01:55 +0000 Subject: [PATCH 2/9] Temporarily add push trigger to ci-nightly.yml for testing Remove before merging. --- .github/workflows/ci-nightly.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci-nightly.yml b/.github/workflows/ci-nightly.yml index c6ecb9fdff8..79e73863fc3 100644 --- a/.github/workflows/ci-nightly.yml +++ b/.github/workflows/ci-nightly.yml @@ -17,6 +17,10 @@ concurrency: cancel-in-progress: true on: + push: + branches: + - "main" + - "pull-request/[0-9]+" schedule: # 2:17 AM UTC daily, after the midnight main CI build finishes. # Avoid minute 0 because GitHub documents high scheduled-workflow load From f1482141ca84c31f468278171037ab0e6a739b43 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 2 Jul 2026 19:41:45 +0000 Subject: [PATCH 3/9] CI: deselect test_get_bar_size_in_kb on gh200 runners The cufile BAR-size query returns CUDA_ERROR_NOT_SUPPORTED on Grace+Hopper (unified memory, no discrete PCIe BAR). Tracked in #2299; remove this deselect once the test skipif is fixed upstream. Uses PYTEST_ADDOPTS so no changes to run-tests are needed. --- .github/workflows/test-wheel-linux.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 982d4d1c491..08e95aba16e 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -351,6 +351,9 @@ jobs: env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + # #2299: BAR-size query returns CUDA_ERROR_NOT_SUPPORTED on G+H; + # skip the test on gh200 runners until upstream cufile guards it. + PYTEST_ADDOPTS: ${{ matrix.GPU == 'gh200' && '--deselect tests/test_cufile.py::test_get_bar_size_in_kb' || '' }} run: run-tests bindings - name: Run cuda.bindings benchmarks (smoke test) From 5bbfa0981b2286d535853b0b5f02bb541e5f1c6a Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Mon, 13 Jul 2026 14:37:34 -0400 Subject: [PATCH 4/9] also add GB300 --- ci/test-matrix.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/test-matrix.yml b/ci/test-matrix.yml index 51fd82f4224..c0044d327a3 100644 --- a/ci/test-matrix.yml +++ b/ci/test-matrix.yml @@ -102,7 +102,8 @@ linux: # nightly-cuda-core (released cuda-core from PyPI against main pathfinder/bindings) - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-cuda-core' } } # nightly-standard (arm64 nightly-only runners — per runner team request) - - { ARCH: 'arm64', PY_VER: '3.14', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'gh200', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-standard' } } + - { ARCH: 'arm64', PY_VER: '3.13', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'gh200', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-standard' } } + - { ARCH: 'arm64', PY_VER: '3.14', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'gb300', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-standard' } } - { ARCH: 'arm64', PY_VER: '3.14', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '2', DRIVER: 'latest', ENV: { MODE: 'nightly-standard' } } - { ARCH: 'arm64', PY_VER: '3.14t', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '2', DRIVER: 'latest', ENV: { MODE: 'nightly-standard' } } From 9362c3a0dafcb11da07b1e804094ac00d7da74f5 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Mon, 13 Jul 2026 21:40:51 +0000 Subject: [PATCH 5/9] cuda.core tests: build saxpy RDC fixture with -arch=all (covers sm_103 / GB300) -arch=all-major only emits X.0 cubins (sm_100, sm_90, ...), so nvJitLink returns ERROR_INVALID_INPUT when the RDC saxpy fixture is linked on GB300 (sm_103): Blackwell no longer serves an sm_10x cubin from the sm_100 image. -arch=all additionally emits sm_103 (and other minor archs) so the linker finds a usable input. The fixture is built in-job by build_test_binaries.py, so there is no committed artifact to regenerate. --- cuda_core/tests/test_binaries/build_test_binaries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuda_core/tests/test_binaries/build_test_binaries.py b/cuda_core/tests/test_binaries/build_test_binaries.py index 5dfa1fe7f08..ca1ceccb268 100644 --- a/cuda_core/tests/test_binaries/build_test_binaries.py +++ b/cuda_core/tests/test_binaries/build_test_binaries.py @@ -37,7 +37,7 @@ def main() -> None: "nvcc", "-dc", *nvcc_extra_flags, - "-arch=all-major", + "-arch=all", "-o", str(object_path), str(source_path), From 6a89eaaaa38d95a57e8c5cca2d8c3798014d5a58 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Mon, 13 Jul 2026 21:45:22 +0000 Subject: [PATCH 6/9] CI: install curand/cublas + cupy for numba-cuda-mlir nightly; extend deselect gate to 0.4.1 test_fortran_contiguous calls cp.random, which dlopens libcurand at runtime; the mlir nightly install didn't provide it, unlike the numba-cuda branch (which installs cuda-toolkit[curand,cublas] + cupy). Mirror that here. Bump the version-gated mlir deselect from <=0.4.0 to <=0.4.1: 0.4.1 still fails the cuobjdump-invoking nvjitlink test (wheel envs lack cuobjdump) and the #135 pytest-contamination tests, so keep them deselected. cu12 rows remain red due to the separate #2320 LTO-IR regression (not in the deselect list). --- .github/workflows/test-wheel-linux.yml | 2 +- .github/workflows/test-wheel-windows.yml | 2 +- ci/tools/run-tests | 12 ++++++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 3e069fc384a..4971d56f832 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -465,7 +465,7 @@ jobs: # tests get run automatically. If they still fail on the newer # version we hear about it loudly (rather than silently masking). DESELECTS=() - if python -c "from packaging.version import Version; import sys; sys.exit(0 if Version('${NUMBA_CUDA_MLIR_VER}') <= Version('0.4.0') else 1)"; then + if python -c "from packaging.version import Version; import sys; sys.exit(0 if Version('${NUMBA_CUDA_MLIR_VER}') <= Version('0.4.1') else 1)"; then # NVIDIA/numba-cuda-mlir#135: serial-pytest contamination of # numba_cuda_mlir.cuda.cudadrv from an xfailed test in # test_nrt_comprehensive.py contaminates any later test that diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 46937cbd12d..e2ff59f39e5 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -443,7 +443,7 @@ jobs: # the full union rather than trying to enumerate what happened # to fail on the most recent nightly. DESELECTS=() - if python -c "from packaging.version import Version; import sys; sys.exit(0 if Version('${NUMBA_CUDA_MLIR_VER}') <= Version('0.4.0') else 1)"; then + if python -c "from packaging.version import Version; import sys; sys.exit(0 if Version('${NUMBA_CUDA_MLIR_VER}') <= Version('0.4.1') else 1)"; then DESELECTS+=( --deselect 'tests/numba_cuda_tests/cudadrv/test_cuda_array_slicing.py::CudaArraySetting::test_no_sync_default_stream' --deselect 'tests/numba_cuda_tests/cudadrv/test_cuda_array_slicing.py::CudaArraySetting::test_no_sync_supplied_stream' diff --git a/ci/tools/run-tests b/ci/tools/run-tests index c5c0cc12336..0920acbae15 100755 --- a/ci/tools/run-tests +++ b/ci/tools/run-tests @@ -162,10 +162,18 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then psutil cffi pytest-xdist pytest-benchmark filecheck ml_dtypes statistics ) elif [[ "${test_module}" == "nightly-numba-cuda-mlir" ]]; then - echo "Installing pathfinder + bindings + core + numba-cuda-mlir" + echo "Installing pathfinder + bindings + core + numba-cuda-mlir + cupy" # numpy<2.5: numba-cuda-mlir 0.4.0 registers np.row_stack, which was # removed in NumPy 2.5. See NVIDIA/numba-cuda-mlir#154. - PIP_ARGS+=("numba-cuda-mlir[cu${TEST_CUDA_MAJOR}]" "numpy<2.5") + # curand/cublas + cupy: some numba-cuda-mlir tests (e.g. + # test_fortran_contiguous) call cp.random, which dlopens libcurand at + # runtime. Mirror the numba-cuda branch, which installs the same libs. + PIP_ARGS+=( + "numba-cuda-mlir[cu${TEST_CUDA_MAJOR}]" + "numpy<2.5" + "cuda-toolkit[curand,cublas]==${CUDA_VER_MINOR}.*" + "cupy-cuda${TEST_CUDA_MAJOR}x" + ) fi pip install "${PIP_ARGS[@]}" From 7f44b44f9df3c4d213c94af48ed8c81377305bb5 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Mon, 13 Jul 2026 17:50:26 -0400 Subject: [PATCH 7/9] nit: align columns --- ci/test-matrix.yml | 54 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/ci/test-matrix.yml b/ci/test-matrix.yml index c0044d327a3..0adb0142ae7 100644 --- a/ci/test-matrix.yml +++ b/ci/test-matrix.yml @@ -93,7 +93,7 @@ linux: - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu130' } } # nightly-numba-cuda - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '580.65.06', ENV: { MODE: 'nightly-numba-cuda' } } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '580.65.06', ENV: { MODE: 'nightly-numba-cuda' } } - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } } - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } } # nightly-numba-cuda-mlir (MLIR backend, linux-64 only) @@ -110,36 +110,36 @@ linux: windows: pull-request: # win-64 - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'rtx2080', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'WDDM' } - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'v100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'rtx4090', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'WDDM' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'rtx4090', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'WDDM' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM', ENV: { CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM: '1' } } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'v100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } - - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'rtx2080', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'WDDM' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } + - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'v100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } + - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'rtx4090', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'WDDM' } + - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'rtx4090', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'WDDM' } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } + - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } + - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } + - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM', ENV: { CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM: '1' } } + - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'v100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } + - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } + - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } + - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC' } + - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } + - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'a100', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } # special runners - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 't4', GPU_COUNT: '2', DRIVER: 'latest', DRIVER_MODE: 'TCC' } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'h100', GPU_COUNT: '2', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } + - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.3.0', LOCAL_CTK: '1', GPU: 't4', GPU_COUNT: '2', DRIVER: 'latest', DRIVER_MODE: 'TCC' } + - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'h100', GPU_COUNT: '2', DRIVER: 'latest', DRIVER_MODE: 'MCDM' } nightly: # nightly-pytorch - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.6.3', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.12.1', TORCH_CUDA: 'cu126' } } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.12.1', TORCH_CUDA: 'cu130' } } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.6.3', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu126' } } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu130' } } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.6.3', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.12.1', TORCH_CUDA: 'cu126' } } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.12.1', TORCH_CUDA: 'cu130' } } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.6.3', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu126' } } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu130' } } # nightly-numba-cuda - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-numba-cuda' } } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '596.36', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-numba-cuda' } } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-numba-cuda' } } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '596.36', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-numba-cuda' } } # nightly-numba-cuda-mlir (MLIR backend, win-64) - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM', ENV: { MODE: 'nightly-numba-cuda-mlir' } } - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.3.0', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM', ENV: { MODE: 'nightly-numba-cuda-mlir' } } From 4b61b3c8ce904df6c4482c1010148f153cdb78a9 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Mon, 13 Jul 2026 22:57:20 +0000 Subject: [PATCH 8/9] Revert "Temporarily add push trigger to ci-nightly.yml for testing" This reverts commit 8d51cf761dda86fc29eeb3e68aba8860e366ef19. --- .github/workflows/ci-nightly.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci-nightly.yml b/.github/workflows/ci-nightly.yml index 42c9e3ed930..0188ebf2524 100644 --- a/.github/workflows/ci-nightly.yml +++ b/.github/workflows/ci-nightly.yml @@ -17,10 +17,6 @@ concurrency: cancel-in-progress: true on: - push: - branches: - - "main" - - "pull-request/[0-9]+" schedule: # 2:17 AM UTC daily, after the midnight main CI build finishes. # Avoid minute 0 because GitHub documents high scheduled-workflow load From a31c2cd278c822ae67cbd264504545118683931b Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Mon, 13 Jul 2026 21:50:21 -0400 Subject: [PATCH 9/9] rename variable --- ci/tools/run-tests | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/tools/run-tests b/ci/tools/run-tests index 0920acbae15..c093ed9e4d2 100755 --- a/ci/tools/run-tests +++ b/ci/tools/run-tests @@ -59,7 +59,7 @@ elif [[ "${test_module}" == "bindings" ]]; then elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then # Shared setup for core and nightly modes. TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${CUDA_VER})" - CUDA_VER_MINOR="$(cut -d '.' -f 1-2 <<< "${CUDA_VER}")" + TEST_CUDA_MAJOR_MINOR="$(cut -d '.' -f 1-2 <<< "${CUDA_VER}")" FREE_THREADING="" if python -c 'import sys; assert not sys._is_gil_enabled()' 2> /dev/null; then @@ -101,7 +101,7 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then echo "Installing core wheel" # Constrain cuda-toolkit to the requested CTK version to avoid # pip pulling in a newer nvidia-cuda-runtime that conflicts with it. - pip install "${CORE_WHL[@]}" --group "test-cu${TEST_CUDA_MAJOR}${FREE_THREADING}" "cuda-toolkit==${CUDA_VER_MINOR}.*" + pip install "${CORE_WHL[@]}" --group "test-cu${TEST_CUDA_MAJOR}${FREE_THREADING}" "cuda-toolkit==${TEST_CUDA_MAJOR_MINOR}.*" echo "Installed packages before core tests:" pip list echo "Running core tests" @@ -145,7 +145,7 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then # Use cuda-toolkit[cudart] only — torch brings its own nvcc/nvrtc/etc. # This avoids version conflicts between our nvidia-* pins and torch's. echo "Installing pathfinder + bindings + core + test deps + PyTorch ${TORCH_VER} (${TORCH_CUDA})" - PIP_ARGS+=("cuda-toolkit[cudart]==${CUDA_VER_MINOR}.*") + PIP_ARGS+=("cuda-toolkit[cudart]==${TEST_CUDA_MAJOR_MINOR}.*") if [[ "${TORCH_VER}" == "latest" ]]; then PIP_ARGS+=(torch) else @@ -156,7 +156,7 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then echo "Installing pathfinder + bindings + core + test deps + numba-cuda" # numba-cuda's test-cuXX group deps (can't use --group for a wheel install): PIP_ARGS+=( - "cuda-toolkit[curand,cublas]==${CUDA_VER_MINOR}.*" + "cuda-toolkit[curand,cublas]==${TEST_CUDA_MAJOR_MINOR}.*" "numba-cuda[cu${TEST_CUDA_MAJOR}]" "cupy-cuda${TEST_CUDA_MAJOR}x" psutil cffi pytest-xdist pytest-benchmark filecheck ml_dtypes statistics @@ -171,7 +171,7 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then PIP_ARGS+=( "numba-cuda-mlir[cu${TEST_CUDA_MAJOR}]" "numpy<2.5" - "cuda-toolkit[curand,cublas]==${CUDA_VER_MINOR}.*" + "cuda-toolkit[curand,cublas]==${TEST_CUDA_MAJOR_MINOR}.*" "cupy-cuda${TEST_CUDA_MAJOR}x" ) fi