diff --git a/.github/workflows/ci-pixi-source-test.yml b/.github/workflows/ci-pixi-source-test.yml index f29dd307e90..869c8c6d8cd 100644 --- a/.github/workflows/ci-pixi-source-test.yml +++ b/.github/workflows/ci-pixi-source-test.yml @@ -154,14 +154,23 @@ jobs: done # The last build was cu13, and each major must have kept its own # generated sources rather than overwriting the other's. - stamp=$(cat cuda_core/build/.build-cuda-major) - if [ "${stamp}" != "13" ]; then - echo "::error::build stamp is '${stamp}', expected 13" + # Stamp is ABI-scoped (.build-config.). Unset and gnu + # both record as "gnu"; llvm records as "llvm". + shopt -s nullglob + stamps=(cuda_core/build/.build-config.*) + if (( ${#stamps[@]} != 1 )); then + echo "::error::expected one ABI stamp, found: ${stamps[*]:-none}" exit 1 fi + stamp=$(<"${stamps[0]}") + if [[ ! "${stamp}" =~ ^cu13-(gnu|llvm)-(debug|opt)(-cov)?$ ]]; then + echo "::error::build stamp is '${stamp}', expected cu13-{gnu|llvm}-{debug|opt}" + exit 1 + fi + suffix="${stamp#cu13-}" for major in cu12 cu13; do - if [ ! -d "cuda_core/build/cython/${major}" ]; then - echo "::error::no ${major} generated-source directory" + if [ ! -d "cuda_core/build/cython/${major}-${suffix}" ]; then + echo "::error::no ${major}-${suffix} generated-source directory" exit 1 fi done diff --git a/cuda_bindings/build_hooks.py b/cuda_bindings/build_hooks.py index 2649fa45e59..a1f572d7a37 100644 --- a/cuda_bindings/build_hooks.py +++ b/cuda_bindings/build_hooks.py @@ -14,6 +14,7 @@ import hashlib import os import re +import shlex import shutil import sys import sysconfig @@ -204,8 +205,42 @@ def _resolve_toolchain_name(): return name, allowed, cc, cxx, explicit +def _with_compiler(command, compiler): + """Replace the leading compiler on a linker command; keep flags. + + Conda ``LDCXXSHARED`` looks like ``g++ -pthread -B .../python_compiler_compat + -shared ...``. Only the executable changes so those flags stay on the + link line. The command is tokenized with shlex so quoted arguments + survive, and a leading ``env VAR=value`` prefix is preserved. CC/CXX are + not rewritten this way: they may already be a launcher plus compiler + (``sccache cc``). + """ + if not command or not command.strip(): + return compiler + parts = shlex.split(command) + # Keep an ``env VAR=value ...`` prefix: setuptools' C++ link step splits it + # off before it substitutes the compiler, so it still reaches the link line. + prefix_end = 0 + if parts and os.path.basename(parts[0]) == "env": + prefix_end = 1 + # Match setuptools' _split_env: any token with ``=`` is an env operand + # (covers both ``VAR=value`` and ``--unset=VAR`` long options). + while prefix_end < len(parts) and "=" in parts[prefix_end]: + prefix_end += 1 + # Everything else before the first flag is the old compiler (or a launcher + # for it; setuptools takes the launcher from CXX instead). + i = prefix_end + while i < len(parts) and not parts[i].startswith("-"): + i += 1 + return shlex.join([*parts[:prefix_end], compiler, *parts[i:]]) + + def _with_sccache(current, compiler): - """Keep CC="sccache cc" as CC="sccache clang" when the toolchain picks a compiler.""" + """Keep a leading sccache token when the toolchain picks a compiler. + + CI sets ``CC="sccache cc"`` or ``CC="/host/.../sccache cc"``. An explicit + toolchain then becomes ``CC="sccache clang"`` rather than a bare compiler. + """ if current: launcher = current.split()[0] if os.path.basename(launcher) == "sccache": @@ -214,18 +249,28 @@ def _with_sccache(current, compiler): def _apply_toolchain_env(cc, cxx, explicit): - """Set CC/CXX/LDSHARED for an explicitly-chosen toolchain. + """Set CC/CXX/LDCXXSHARED for an explicitly-chosen toolchain. The default path (CUDA_PYTHON_TOOLCHAIN unset) intentionally does not touch the env, so an externally-set compiler (e.g. CC="sccache cc" in CI) keeps working. An explicit CUDA_PYTHON_TOOLCHAIN - override (incl. =gnu) governs the compiler. An existing sccache prefix - is kept (CC="sccache cc" + llvm -> CC="sccache clang"). + override (incl. =gnu) sets CC/CXX to the toolchain compiler; an existing + sccache prefix is kept (CC="sccache cc" + llvm -> CC="sccache clang"). + Extras on LDCXXSHARED (rpath, -pthread, -B, ...) are kept, taken from the + environment if set there and from sysconfig otherwise; only the compiler + is swapped. LDSHARED is left unset so distutils rewrites it from CC. """ if explicit and cc is not None: os.environ["CC"] = _with_sccache(os.environ.get("CC", ""), cc) os.environ["CXX"] = _with_sccache(os.environ.get("CXX", ""), cxx) - os.environ["LDSHARED"] = f"{os.environ['CXX']} -shared" + # An LDCXXSHARED the user already exported takes precedence over + # sysconfig's, as CC/CXX do; either way only the compiler is swapped. + ldcxxshared = ( + os.environ.get("LDCXXSHARED") + or sysconfig.get_config_var("LDCXXSHARED") + or sysconfig.get_config_var("LDSHARED") + ) + os.environ["LDCXXSHARED"] = _with_compiler(ldcxxshared, cxx) if ldcxxshared else f"{cxx} -shared" def _check_toolchain_available(name): @@ -381,9 +426,9 @@ def _resolve_toolchain(debug=False, compile_for_coverage=False): Returns (name, cc, cxx, extra_compile_args, extra_link_args). The default toolchain (gnu on Linux, msvc on Windows) reproduces the previous build - behavior and does not touch CC/CXX/LDSHARED, so an externally-set compiler + behavior and does not touch CC/CXX/LDCXXSHARED, so an externally-set compiler (e.g. CC="sccache cc") keeps working. A non-default toolchain (llvm on - Linux) selects clang/clang++ and lld and sets CC/CXX/LDSHARED so distutils' + Linux) selects clang/clang++ and lld and sets CC/CXX/LDCXXSHARED so distutils' customize_compiler picks them up. """ name, _allowed, cc, cxx, explicit = _resolve_toolchain_name() diff --git a/cuda_bindings/tests/test_build_hooks.py b/cuda_bindings/tests/test_build_hooks.py index 0c59e532d29..78c41ed1885 100644 --- a/cuda_bindings/tests/test_build_hooks.py +++ b/cuda_bindings/tests/test_build_hooks.py @@ -16,7 +16,9 @@ import importlib.util import os +import shlex import sys +import sysconfig from pathlib import Path # build_hooks.py imports Cython and setuptools at the top level; both are @@ -25,6 +27,8 @@ import Cython # noqa: F401 import pytest import setuptools # noqa: F401 +from setuptools._distutils.ccompiler import new_compiler +from setuptools._distutils.sysconfig import customize_compiler def _load_build_hooks(): @@ -41,7 +45,7 @@ def _load_build_hooks(): @pytest.fixture(autouse=True) def _isolate_toolchain_env(): - names = ("CUDA_PYTHON_TOOLCHAIN", "CC", "CXX", "LDSHARED", "CUDA_PYTHON_CYTHON_CACHE_DIR") + names = ("CUDA_PYTHON_TOOLCHAIN", "CC", "CXX", "LDSHARED", "LDCXXSHARED", "CUDA_PYTHON_CYTHON_CACHE_DIR") original = {name: os.environ[name] for name in names if name in os.environ} for name in names: os.environ.pop(name, None) @@ -53,11 +57,16 @@ def _isolate_toolchain_env(): os.environ.update(original) +def _fake_sysconfig(monkeypatch, **values): + """Pin sysconfig.get_config_var so linker-command assertions are exact.""" + monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda name: values.get(name)) + + class TestResolveToolchain: """_resolve_toolchain: pick compiler/linker/flags from CUDA_PYTHON_TOOLCHAIN. The default toolchain (gnu on Linux, msvc on Windows) must reproduce the - previous build behavior exactly and must not touch CC/CXX/LDSHARED, so an + previous build behavior exactly and must not touch CC/CXX/LDCXXSHARED, so an externally-set compiler (e.g. the sccache wrapper in CI) survives. """ @@ -67,6 +76,7 @@ def test_default_does_not_touch_env(self, monkeypatch): monkeypatch.delenv("CC", raising=False) monkeypatch.delenv("CXX", raising=False) monkeypatch.delenv("LDSHARED", raising=False) + monkeypatch.delenv("LDCXXSHARED", raising=False) name, cc, cxx, _cargs, _largs = build_hooks._resolve_toolchain() if sys.platform == "win32": assert name == "msvc" @@ -75,6 +85,7 @@ def test_default_does_not_touch_env(self, monkeypatch): assert name == "gnu" assert (cc, cxx) == ("gcc", "g++") assert "CC" not in os.environ and "CXX" not in os.environ + assert "LDCXXSHARED" not in os.environ @pytest.mark.agent_authored(model="glm-5.2") def test_default_preserves_existing_cc(self, monkeypatch): @@ -105,6 +116,7 @@ def test_llvm_sets_env_and_flags(self, monkeypatch): if sys.platform == "win32": pytest.skip("llvm only valid on Linux") monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") monkeypatch.delenv("CC", raising=False) monkeypatch.delenv("CXX", raising=False) monkeypatch.delenv("LDSHARED", raising=False) @@ -113,6 +125,8 @@ def test_llvm_sets_env_and_flags(self, monkeypatch): assert (cc, cxx) == ("clang", "clang++") assert os.environ["CC"] == "clang" assert os.environ["CXX"] == "clang++" + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" + assert "LDSHARED" not in os.environ assert "-fuse-ld=lld" in largs # clang rejects the gcc-only flags that gnu uses; they must be absent. assert "-fpermissive" not in cargs @@ -123,6 +137,7 @@ def test_gnu_sets_env_and_flags(self, monkeypatch): if sys.platform == "win32": pytest.skip("gnu only valid on Linux") monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "gnu") + _fake_sysconfig(monkeypatch, LDCXXSHARED="x86_64-linux-gnu-g++ -shared -Wl,-O1") monkeypatch.delenv("CC", raising=False) monkeypatch.delenv("CXX", raising=False) monkeypatch.delenv("LDSHARED", raising=False) @@ -131,6 +146,8 @@ def test_gnu_sets_env_and_flags(self, monkeypatch): assert (cc, cxx) == ("gcc", "g++") assert os.environ["CC"] == "gcc" assert os.environ["CXX"] == "g++" + assert os.environ["LDCXXSHARED"] == "g++ -shared -Wl,-O1" + assert "LDSHARED" not in os.environ # gcc-only flags are present (this is the point of P2: explicit gnu must use gcc, not generic cc) assert "-fpermissive" in cargs assert "-fno-var-tracking-assignments" in cargs @@ -140,12 +157,15 @@ def test_llvm_keeps_sccache_prefix(self, monkeypatch): if sys.platform == "win32": pytest.skip("llvm only valid on Linux") monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") monkeypatch.setenv("CC", "sccache cc") monkeypatch.setenv("CXX", "sccache c++") _name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() assert os.environ["CC"] == "sccache clang" assert os.environ["CXX"] == "sccache clang++" - assert os.environ["LDSHARED"] == "sccache clang++ -shared" + assert "LDSHARED" not in os.environ + # The launcher prefixes CC/CXX only; the shared linker command is the bare compiler. + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" @pytest.mark.agent_authored(model="glm-5.2") def test_gnu_keeps_gcc_only_flags(self, monkeypatch): @@ -156,6 +176,164 @@ def test_gnu_keeps_gcc_only_flags(self, monkeypatch): assert "-fpermissive" in cargs assert "-fno-var-tracking-assignments" in cargs + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_explicit_toolchain_prefers_env_ldcxxshared(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("gnu/llvm only valid on Linux") + _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") + monkeypatch.setenv("LDCXXSHARED", "g++ -shared -Wl,-rpath,/user/lib") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + build_hooks._resolve_toolchain() + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-rpath,/user/lib" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_explicit_toolchain_falls_back_to_ldshared_then_shared_flag(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + _fake_sysconfig(monkeypatch, LDSHARED="gcc -shared -Wl,-z,relro") + build_hooks._resolve_toolchain() + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-z,relro" + _fake_sysconfig(monkeypatch) + os.environ.pop("LDCXXSHARED") + build_hooks._resolve_toolchain() + assert os.environ["LDCXXSHARED"] == "clang++ -shared" + + +class TestWithSccache: + """_with_sccache: keep a leading sccache token, swap the compiler.""" + + @pytest.mark.agent_authored(model="grok-4.6") + def test_keeps_sccache_and_swaps_compiler(self): + assert build_hooks._with_sccache("sccache cc", "clang") == "sccache clang" + + @pytest.mark.agent_authored(model="grok-4.6") + def test_keeps_absolute_sccache_path(self): + assert ( + build_hooks._with_sccache("/host/usr/local/bin/sccache cc", "clang") == "/host/usr/local/bin/sccache clang" + ) + + @pytest.mark.agent_authored(model="grok-4.6") + def test_bare_or_unrelated_cc_returns_compiler(self): + assert build_hooks._with_sccache("", "clang") == "clang" + assert build_hooks._with_sccache("gcc", "clang") == "clang" + assert build_hooks._with_sccache("ccache gcc", "clang") == "clang" + + +class TestWithCompiler: + """_with_compiler: replace the compiler executable, keep following flags.""" + + @pytest.mark.agent_authored(model="grok-4.6") + def test_keeps_flags_that_were_part_of_sysconfig_cxx(self): + assert ( + build_hooks._with_compiler("g++ -pthread -B /compat -shared -Wl,-rpath,/lib", "clang++") + == "clang++ -pthread -B /compat -shared -Wl,-rpath,/lib" + ) + + @pytest.mark.agent_authored(model="grok-4.6") + def test_compiler_only_command_returns_compiler(self): + assert build_hooks._with_compiler("g++", "clang++") == "clang++" + assert build_hooks._with_compiler("", "clang++") == "clang++" + assert build_hooks._with_compiler(None, "clang++") == "clang++" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_keeps_quoted_arguments_intact(self): + result = build_hooks._with_compiler("g++ -Wl,-rpath='/a b' -shared", "clang++") + assert shlex.split(result) == ["clang++", "-Wl,-rpath=/a b", "-shared"] + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_drops_launcher_before_compiler(self): + # setuptools takes the launcher from CXX; keeping a second copy here would + # leave a stray compiler argument on the link line. + assert build_hooks._with_compiler("ccache g++ -shared", "clang++") == "clang++ -shared" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_keeps_env_prefix(self): + assert ( + build_hooks._with_compiler("env LIBRARY_PATH=/custom/lib g++ -shared", "clang++") + == "env LIBRARY_PATH=/custom/lib clang++ -shared" + ) + # env long options (--unset=VAR) also treated as prefix, same as setuptools' _split_env + assert ( + build_hooks._with_compiler("env --unset=LD_LIBRARY_PATH g++ -shared", "clang++") + == "env --unset=LD_LIBRARY_PATH clang++ -shared" + ) + + +class TestDistutilsLinkerIntegration: + """The env set by _resolve_toolchain, as setuptools' distutils consumes it. + + The tests above check os.environ; this checks the linker commands distutils + derives from it, which is what actually reaches the C++ link step. + """ + + @staticmethod + def _customized_compiler(): + compiler = new_compiler() + customize_compiler(compiler) + if not hasattr(compiler, "linker_so_cxx"): + pytest.skip("this setuptools' distutils has no linker_so_cxx") + return compiler + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linker_so_cxx_swaps_compiler_and_keeps_sysconfig_flags(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + sysconfig_ld = sysconfig.get_config_var("LDCXXSHARED") + if not sysconfig_ld: + pytest.skip("this Python has no LDCXXSHARED") + # Everything from the first flag on, including operands such as ``-B /path``. + tokens = shlex.split(sysconfig_ld) + first_flag = next((i for i, tok in enumerate(tokens) if tok.startswith("-")), len(tokens)) + expected_tail = tokens[first_flag:] + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + build_hooks._resolve_toolchain() + linker = self._customized_compiler().linker_so_cxx + assert linker[0] == "clang++" + assert linker[1 : 1 + len(expected_tail)] == expected_tail + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linker_so_cxx_keeps_split_option_operands(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("LDCXXSHARED", "g++ -pthread -B /path/to/python_compiler_compat -shared -Wl,-rpath,/lib") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + build_hooks._resolve_toolchain() + linker = self._customized_compiler().linker_so_cxx + assert linker[:6] == [ + "clang++", + "-pthread", + "-B", + "/path/to/python_compiler_compat", + "-shared", + "-Wl,-rpath,/lib", + ] + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linker_so_cxx_keeps_env_prefix(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("LDCXXSHARED", "env LIBRARY_PATH=/custom/lib g++ -shared") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + build_hooks._resolve_toolchain() + linker = self._customized_compiler().linker_so_cxx + assert linker[:3] == ["env", "LIBRARY_PATH=/custom/lib", "clang++"] + assert "g++" not in linker + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_sccache_does_not_duplicate_compiler_in_cxx_linker(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + monkeypatch.setenv("CC", "sccache cc") + monkeypatch.setenv("CXX", "sccache c++") + build_hooks._resolve_toolchain() + compiler = self._customized_compiler() + assert compiler.compiler_cxx[:2] == ["sccache", "clang++"] + linker = compiler.linker_so_cxx + assert linker[0] == "clang++" + assert linker.count("clang++") == 1 + class TestCheckToolchainAvailable: """_check_toolchain_available: fast, helpful failure when a tool is missing.""" diff --git a/cuda_core/build_hooks.py b/cuda_core/build_hooks.py index c977f998e58..afa956fa4d1 100644 --- a/cuda_core/build_hooks.py +++ b/cuda_core/build_hooks.py @@ -14,6 +14,7 @@ import importlib.util import os import re +import shlex import shutil import sys import sysconfig @@ -251,8 +252,42 @@ def _resolve_toolchain_name(): return name, allowed, cc, cxx, explicit +def _with_compiler(command, compiler): + """Replace the leading compiler on a linker command; keep flags. + + Conda ``LDCXXSHARED`` looks like ``g++ -pthread -B .../python_compiler_compat + -shared ...``. Only the executable changes so those flags stay on the + link line. The command is tokenized with shlex so quoted arguments + survive, and a leading ``env VAR=value`` prefix is preserved. CC/CXX are + not rewritten this way: they may already be a launcher plus compiler + (``sccache cc``). + """ + if not command or not command.strip(): + return compiler + parts = shlex.split(command) + # Keep an ``env VAR=value ...`` prefix: setuptools' C++ link step splits it + # off before it substitutes the compiler, so it still reaches the link line. + prefix_end = 0 + if parts and os.path.basename(parts[0]) == "env": + prefix_end = 1 + # Match setuptools' _split_env: any token with ``=`` is an env operand + # (covers both ``VAR=value`` and ``--unset=VAR`` long options). + while prefix_end < len(parts) and "=" in parts[prefix_end]: + prefix_end += 1 + # Everything else before the first flag is the old compiler (or a launcher + # for it; setuptools takes the launcher from CXX instead). + i = prefix_end + while i < len(parts) and not parts[i].startswith("-"): + i += 1 + return shlex.join([*parts[:prefix_end], compiler, *parts[i:]]) + + def _with_sccache(current, compiler): - """Keep CC="sccache cc" as CC="sccache clang" when the toolchain picks a compiler.""" + """Keep a leading sccache token when the toolchain picks a compiler. + + CI sets ``CC="sccache cc"`` or ``CC="/host/.../sccache cc"``. An explicit + toolchain then becomes ``CC="sccache clang"`` rather than a bare compiler. + """ if current: launcher = current.split()[0] if os.path.basename(launcher) == "sccache": @@ -261,18 +296,28 @@ def _with_sccache(current, compiler): def _apply_toolchain_env(cc, cxx, explicit): - """Set CC/CXX/LDSHARED for an explicitly-chosen toolchain. + """Set CC/CXX/LDCXXSHARED for an explicitly-chosen toolchain. The default path (CUDA_PYTHON_TOOLCHAIN unset) intentionally does not touch the env, so an externally-set compiler (e.g. CC="sccache cc" in CI) keeps working. An explicit CUDA_PYTHON_TOOLCHAIN - override (incl. =gnu) governs the compiler. An existing sccache prefix - is kept (CC="sccache cc" + llvm -> CC="sccache clang"). + override (incl. =gnu) sets CC/CXX to the toolchain compiler; an existing + sccache prefix is kept (CC="sccache cc" + llvm -> CC="sccache clang"). + Extras on LDCXXSHARED (rpath, -pthread, -B, ...) are kept, taken from the + environment if set there and from sysconfig otherwise; only the compiler + is swapped. LDSHARED is left unset so distutils rewrites it from CC. """ if explicit and cc is not None: os.environ["CC"] = _with_sccache(os.environ.get("CC", ""), cc) os.environ["CXX"] = _with_sccache(os.environ.get("CXX", ""), cxx) - os.environ["LDSHARED"] = f"{os.environ['CXX']} -shared" + # An LDCXXSHARED the user already exported takes precedence over + # sysconfig's, as CC/CXX do; either way only the compiler is swapped. + ldcxxshared = ( + os.environ.get("LDCXXSHARED") + or sysconfig.get_config_var("LDCXXSHARED") + or sysconfig.get_config_var("LDSHARED") + ) + os.environ["LDCXXSHARED"] = _with_compiler(ldcxxshared, cxx) if ldcxxshared else f"{cxx} -shared" def _check_toolchain_available(name): @@ -428,9 +473,9 @@ def _resolve_toolchain(debug=False, compile_for_coverage=False): Returns (name, cc, cxx, extra_compile_args, extra_link_args). The default toolchain (gnu on Linux, msvc on Windows) reproduces the previous build - behavior and does not touch CC/CXX/LDSHARED, so an externally-set compiler + behavior and does not touch CC/CXX/LDCXXSHARED, so an externally-set compiler (e.g. CC="sccache cc") keeps working. A non-default toolchain (llvm on - Linux) selects clang/clang++ and lld and sets CC/CXX/LDSHARED so distutils' + Linux) selects clang/clang++ and lld and sets CC/CXX/LDCXXSHARED so distutils' customize_compiler picks them up. """ name, _allowed, cc, cxx, explicit = _resolve_toolchain_name() diff --git a/cuda_core/tests/test_build_hooks.py b/cuda_core/tests/test_build_hooks.py index 0e948bb087d..7e9499f89e3 100644 --- a/cuda_core/tests/test_build_hooks.py +++ b/cuda_core/tests/test_build_hooks.py @@ -19,7 +19,9 @@ import builtins import importlib.util import os +import shlex import sys +import sysconfig import tempfile import threading import types @@ -33,6 +35,8 @@ import Cython # noqa: F401 import pytest import setuptools # noqa: F401 +from setuptools._distutils.ccompiler import new_compiler +from setuptools._distutils.sysconfig import customize_compiler from cuda.pathfinder import get_cuda_path_or_home @@ -58,7 +62,7 @@ def _load_build_hooks(): @pytest.fixture(autouse=True) def _isolate_toolchain_env(): - names = ("CUDA_PYTHON_TOOLCHAIN", "CC", "CXX", "LDSHARED", "CUDA_PYTHON_CYTHON_CACHE_DIR") + names = ("CUDA_PYTHON_TOOLCHAIN", "CC", "CXX", "LDSHARED", "LDCXXSHARED", "CUDA_PYTHON_CYTHON_CACHE_DIR") original = {name: os.environ[name] for name in names if name in os.environ} for name in names: os.environ.pop(name, None) @@ -70,6 +74,11 @@ def _isolate_toolchain_env(): os.environ.update(original) +def _fake_sysconfig(monkeypatch, **values): + """Pin sysconfig.get_config_var so linker-command assertions are exact.""" + monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda name: values.get(name)) + + @pytest.mark.agent_authored(model="gpt-5.6") def test_cuda_path_is_resolved_before_importing_bindings(monkeypatch): """PEP 517 namespace repair runs before cuda.bindings is imported.""" @@ -885,7 +894,7 @@ class TestResolveToolchain: """_resolve_toolchain: pick compiler/linker/flags from CUDA_PYTHON_TOOLCHAIN. The default toolchain (gnu on Linux, msvc on Windows) must reproduce the - previous build behavior exactly and must not touch CC/CXX/LDSHARED, so an + previous build behavior exactly and must not touch CC/CXX/LDCXXSHARED, so an externally-set compiler (e.g. the sccache wrapper in CI) survives. """ @@ -895,6 +904,7 @@ def test_default_does_not_touch_env(self, monkeypatch): monkeypatch.delenv("CC", raising=False) monkeypatch.delenv("CXX", raising=False) monkeypatch.delenv("LDSHARED", raising=False) + monkeypatch.delenv("LDCXXSHARED", raising=False) name, cc, cxx, _cargs, _largs = build_hooks._resolve_toolchain() if sys.platform == "win32": assert name == "msvc" @@ -903,6 +913,7 @@ def test_default_does_not_touch_env(self, monkeypatch): assert name == "gnu" assert (cc, cxx) == ("gcc", "g++") assert "CC" not in os.environ and "CXX" not in os.environ + assert "LDCXXSHARED" not in os.environ @pytest.mark.agent_authored(model="glm-5.2") def test_default_preserves_existing_cc(self, monkeypatch): @@ -933,6 +944,7 @@ def test_llvm_sets_env_and_flags(self, monkeypatch): if sys.platform == "win32": pytest.skip("llvm only valid on Linux") monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") monkeypatch.delenv("CC", raising=False) monkeypatch.delenv("CXX", raising=False) monkeypatch.delenv("LDSHARED", raising=False) @@ -941,6 +953,8 @@ def test_llvm_sets_env_and_flags(self, monkeypatch): assert (cc, cxx) == ("clang", "clang++") assert os.environ["CC"] == "clang" assert os.environ["CXX"] == "clang++" + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" + assert "LDSHARED" not in os.environ assert "-fuse-ld=lld" in largs # clang rejects the gcc-only flags that gnu uses; they must be absent. assert "-fpermissive" not in cargs @@ -951,6 +965,7 @@ def test_gnu_sets_env_and_flags(self, monkeypatch): if sys.platform == "win32": pytest.skip("gnu only valid on Linux") monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "gnu") + _fake_sysconfig(monkeypatch, LDCXXSHARED="x86_64-linux-gnu-g++ -shared -Wl,-O1") monkeypatch.delenv("CC", raising=False) monkeypatch.delenv("CXX", raising=False) monkeypatch.delenv("LDSHARED", raising=False) @@ -959,6 +974,8 @@ def test_gnu_sets_env_and_flags(self, monkeypatch): assert (cc, cxx) == ("gcc", "g++") assert os.environ["CC"] == "gcc" assert os.environ["CXX"] == "g++" + assert os.environ["LDCXXSHARED"] == "g++ -shared -Wl,-O1" + assert "LDSHARED" not in os.environ # gcc-only flags are present (this is the point of P2: explicit gnu must use gcc, not generic cc) assert "-fpermissive" not in cargs # cuda.core gnu flags don't include it; bindings do assert "-fno-var-tracking-assignments" not in cargs @@ -968,12 +985,173 @@ def test_llvm_keeps_sccache_prefix(self, monkeypatch): if sys.platform == "win32": pytest.skip("llvm only valid on Linux") monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") monkeypatch.setenv("CC", "sccache cc") monkeypatch.setenv("CXX", "sccache c++") _name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() assert os.environ["CC"] == "sccache clang" assert os.environ["CXX"] == "sccache clang++" - assert os.environ["LDSHARED"] == "sccache clang++ -shared" + assert "LDSHARED" not in os.environ + # The launcher prefixes CC/CXX only; the shared linker command is the bare compiler. + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_explicit_toolchain_prefers_env_ldcxxshared(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("gnu/llvm only valid on Linux") + _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") + monkeypatch.setenv("LDCXXSHARED", "g++ -shared -Wl,-rpath,/user/lib") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + build_hooks._resolve_toolchain() + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-rpath,/user/lib" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_explicit_toolchain_falls_back_to_ldshared_then_shared_flag(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + _fake_sysconfig(monkeypatch, LDSHARED="gcc -shared -Wl,-z,relro") + build_hooks._resolve_toolchain() + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-z,relro" + _fake_sysconfig(monkeypatch) + os.environ.pop("LDCXXSHARED") + build_hooks._resolve_toolchain() + assert os.environ["LDCXXSHARED"] == "clang++ -shared" + + +class TestWithSccache: + """_with_sccache: keep a leading sccache token, swap the compiler.""" + + @pytest.mark.agent_authored(model="grok-4.6") + def test_keeps_sccache_and_swaps_compiler(self): + assert build_hooks._with_sccache("sccache cc", "clang") == "sccache clang" + + @pytest.mark.agent_authored(model="grok-4.6") + def test_keeps_absolute_sccache_path(self): + assert ( + build_hooks._with_sccache("/host/usr/local/bin/sccache cc", "clang") == "/host/usr/local/bin/sccache clang" + ) + + @pytest.mark.agent_authored(model="grok-4.6") + def test_bare_or_unrelated_cc_returns_compiler(self): + assert build_hooks._with_sccache("", "clang") == "clang" + assert build_hooks._with_sccache("gcc", "clang") == "clang" + assert build_hooks._with_sccache("ccache gcc", "clang") == "clang" + + +class TestWithCompiler: + """_with_compiler: replace the compiler executable, keep following flags.""" + + @pytest.mark.agent_authored(model="grok-4.6") + def test_keeps_flags_that_were_part_of_sysconfig_cxx(self): + assert ( + build_hooks._with_compiler("g++ -pthread -B /compat -shared -Wl,-rpath,/lib", "clang++") + == "clang++ -pthread -B /compat -shared -Wl,-rpath,/lib" + ) + + @pytest.mark.agent_authored(model="grok-4.6") + def test_compiler_only_command_returns_compiler(self): + assert build_hooks._with_compiler("g++", "clang++") == "clang++" + assert build_hooks._with_compiler("", "clang++") == "clang++" + assert build_hooks._with_compiler(None, "clang++") == "clang++" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_keeps_quoted_arguments_intact(self): + result = build_hooks._with_compiler("g++ -Wl,-rpath='/a b' -shared", "clang++") + assert shlex.split(result) == ["clang++", "-Wl,-rpath=/a b", "-shared"] + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_drops_launcher_before_compiler(self): + # setuptools takes the launcher from CXX; keeping a second copy here would + # leave a stray compiler argument on the link line. + assert build_hooks._with_compiler("ccache g++ -shared", "clang++") == "clang++ -shared" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_keeps_env_prefix(self): + assert ( + build_hooks._with_compiler("env LIBRARY_PATH=/custom/lib g++ -shared", "clang++") + == "env LIBRARY_PATH=/custom/lib clang++ -shared" + ) + # env long options (--unset=VAR) also treated as prefix, same as setuptools' _split_env + assert ( + build_hooks._with_compiler("env --unset=LD_LIBRARY_PATH g++ -shared", "clang++") + == "env --unset=LD_LIBRARY_PATH clang++ -shared" + ) + + +class TestDistutilsLinkerIntegration: + """The env set by _resolve_toolchain, as setuptools' distutils consumes it. + + The tests above check os.environ; this checks the linker commands distutils + derives from it, which is what actually reaches the C++ link step. + """ + + @staticmethod + def _customized_compiler(): + compiler = new_compiler() + customize_compiler(compiler) + if not hasattr(compiler, "linker_so_cxx"): + pytest.skip("this setuptools' distutils has no linker_so_cxx") + return compiler + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linker_so_cxx_swaps_compiler_and_keeps_sysconfig_flags(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + sysconfig_ld = sysconfig.get_config_var("LDCXXSHARED") + if not sysconfig_ld: + pytest.skip("this Python has no LDCXXSHARED") + # Everything from the first flag on, including operands such as ``-B /path``. + tokens = shlex.split(sysconfig_ld) + first_flag = next((i for i, tok in enumerate(tokens) if tok.startswith("-")), len(tokens)) + expected_tail = tokens[first_flag:] + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + build_hooks._resolve_toolchain() + linker = self._customized_compiler().linker_so_cxx + assert linker[0] == "clang++" + assert linker[1 : 1 + len(expected_tail)] == expected_tail + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linker_so_cxx_keeps_split_option_operands(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("LDCXXSHARED", "g++ -pthread -B /path/to/python_compiler_compat -shared -Wl,-rpath,/lib") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + build_hooks._resolve_toolchain() + linker = self._customized_compiler().linker_so_cxx + assert linker[:6] == [ + "clang++", + "-pthread", + "-B", + "/path/to/python_compiler_compat", + "-shared", + "-Wl,-rpath,/lib", + ] + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linker_so_cxx_keeps_env_prefix(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("LDCXXSHARED", "env LIBRARY_PATH=/custom/lib g++ -shared") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + build_hooks._resolve_toolchain() + linker = self._customized_compiler().linker_so_cxx + assert linker[:3] == ["env", "LIBRARY_PATH=/custom/lib", "clang++"] + assert "g++" not in linker + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_sccache_does_not_duplicate_compiler_in_cxx_linker(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + monkeypatch.setenv("CC", "sccache cc") + monkeypatch.setenv("CXX", "sccache c++") + build_hooks._resolve_toolchain() + compiler = self._customized_compiler() + assert compiler.compiler_cxx[:2] == ["sccache", "clang++"] + linker = compiler.linker_so_cxx + assert linker[0] == "clang++" + assert linker.count("clang++") == 1 class TestCheckToolchainAvailable: