diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15924c3073d..865dc9fe65a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -633,6 +633,9 @@ jobs: permissions: contents: read steps: + - name: Enable Git symlinks (Windows, must precede checkout) + run: git config --global core.symlinks true + - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ad23d0e48a4..6dc6c3ff877 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -232,6 +232,9 @@ jobs: CUDA_PYTHON_COVERAGE: "1" CUDA_VER: ${{ needs.coverage-vars.outputs.CUDA_VER }} steps: + - name: Enable Git symlinks (Windows, must precede checkout) + run: git config --global core.symlinks true + - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -348,6 +351,9 @@ jobs: run: shell: bash --noprofile --norc -xeuo pipefail {0} steps: + - name: Enable Git symlinks (Windows, must precede checkout) + run: git config --global core.symlinks true + - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ebad9fb9097..640a4d4ff66 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -81,13 +81,6 @@ repos: language: python files: '^cuda_core/tests/.*\.py$' - - id: check-build-hooks-sync - name: Check shared build_hooks.py blocks are in sync - entry: python ./toolshed/check_build_hooks_sync.py - language: python - files: '^(cuda_bindings|cuda_core)/build_hooks\.py$' - pass_filenames: false - - id: no-markdown-in-docs-source name: Prevent markdown files in docs/source directories entry: bash -c diff --git a/cuda_bindings/AGENTS.md b/cuda_bindings/AGENTS.md index f0e4d9988c4..e9e900c2185 100644 --- a/cuda_bindings/AGENTS.md +++ b/cuda_bindings/AGENTS.md @@ -19,7 +19,10 @@ subpackage in the `cuda-python` monorepo. - **Platform internals**: `cuda/bindings/_internal/` contains platform-specific implementation files and support code. - **Build backend**: `build_hooks.py` drives extension configuration and - Cythonization. + Cythonization. Logic shared with `cuda_core` (toolchain selection, the + compiler flag set, the Cython cache helpers and the rebuild stamps) lives + in `_build_shared.py`; `cuda_core/_build_shared.py` is a symlink to this + file, so an edit here changes both packages. ## Generated-source workflow diff --git a/cuda_bindings/MANIFEST.in b/cuda_bindings/MANIFEST.in index 4394e5d9fe3..6d01549d3e2 100644 --- a/cuda_bindings/MANIFEST.in +++ b/cuda_bindings/MANIFEST.in @@ -6,3 +6,5 @@ recursive-include cuda/ *.pyx *.pxd *.pxi *.pyx.in *.pxd.in *.pxi.in *.h # to the payload, causing file copying to the build environment failed exclude cuda/bindings cuda?bindings exclude cuda/bindings/_bindings cuda?bindings?_bindings +# canonical shared PEP 517 helper (cuda_core/_build_shared.py symlinks to the cuda_bindings copy) +include _build_shared.py diff --git a/cuda_bindings/_build_shared.py b/cuda_bindings/_build_shared.py new file mode 100644 index 00000000000..9fef4c91b5c --- /dev/null +++ b/cuda_bindings/_build_shared.py @@ -0,0 +1,486 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build helpers shared by the cuda-bindings and cuda-core PEP 517 backends. + +This is the single source of truth. ``cuda_core/_build_shared.py`` is a symlink +to this file. Python does not dereference symlinks in ``__file__``, so any +``Path(__file__)``-relative location in here resolves under whichever package +loads it. + +PEP 517 build isolation gives each backend its own ``build_hooks.py`` but not a +shared import path. Both packages declare ``backend-path = ["."]``, which puts +the package directory (and therefore this module) on ``sys.path`` while the +backend runs. + +Only what genuinely differs per package is parameterized, via the arguments of +``resolve_toolchain``: the C++ standard, warnings-as-errors, and an optional +``tweak`` hook for flags a single package needs. + +Besides the toolchain, this module owns the CUDA path lookup and the machinery that both backends use +around cythonize and build_ext: the opt-in Cython generated-source cache and +the build stamps that force a rebuild when the build configuration changed. + +Note: There is no support guarantee for environment variables like +CUDA_PYTHON_TOOLCHAIN. They may be removed or changed in the future. +""" + +import contextlib +import functools +import hashlib +import os +import shlex +import shutil +import sys +import sysconfig +import uuid +from pathlib import Path +from warnings import warn + +# ----------------------------------------------------------------------- +# CUDA path + + +def _import_get_cuda_path_or_home(): + """Import get_cuda_path_or_home, working around PEP 517 namespace shadowing. + + See https://github.com/NVIDIA/cuda-python/issues/1824 for why this helper is needed. + """ + try: + import cuda.pathfinder + except ModuleNotFoundError as exc: + if exc.name not in ("cuda", "cuda.pathfinder"): + raise + try: + import cuda + except ModuleNotFoundError: + cuda = None + + for p in sys.path: + sp_cuda = Path(p) / "cuda" + if (sp_cuda / "pathfinder").is_dir(): + cuda.__path__ = list(cuda.__path__) + [str(sp_cuda)] + break + else: + raise ModuleNotFoundError( + "cuda-pathfinder is not installed in the build environment. " + "Ensure 'cuda-pathfinder>=1.5' is in build-system.requires." + ) + import cuda.pathfinder + + pathfinder_dir = Path(cuda.pathfinder.__file__).parent + print( + f"Using cuda-pathfinder {cuda.pathfinder.__version__} from {pathfinder_dir}", + file=sys.stderr, + ) + return cuda.pathfinder.get_cuda_path_or_home + + +@functools.cache +def _get_cuda_path() -> str: + get_cuda_path_or_home = _import_get_cuda_path_or_home() + cuda_path = get_cuda_path_or_home() + if not cuda_path: + raise RuntimeError("Environment variable CUDA_PATH or CUDA_HOME is not set") + print("CUDA path:", cuda_path) + return cuda_path + + +# ----------------------------------------------------------------------- +# Toolchain selection + +_TOOLCHAINS_LINUX = ("gnu", "llvm") +_TOOLCHAINS_WINDOWS = ("msvc",) +_TOOLCHAIN_COMPILERS = { + "gnu": ("gcc", "g++"), + "llvm": ("clang", "clang++"), + "msvc": (None, None), +} + + +def _resolve_toolchain_name(): + """Read CUDA_PYTHON_TOOLCHAIN, validate it, return (name, allowed, cc, cxx). + + The default toolchain (gnu on Linux, msvc on Windows) is the first entry + of the platform's allowed tuple. cc/cxx are the compiler binaries for the + toolchain (None for msvc, which distutils discovers via the MSVC env). + """ + if sys.platform == "win32": + platform_key, allowed = "win32", _TOOLCHAINS_WINDOWS + else: + platform_key, allowed = "linux", _TOOLCHAINS_LINUX + name = os.environ.get("CUDA_PYTHON_TOOLCHAIN", allowed[0]).strip().lower() + if name not in allowed: + raise RuntimeError( + f"CUDA_PYTHON_TOOLCHAIN={name!r} is not supported on {platform_key}. Valid values: {', '.join(allowed)}." + ) + cc, cxx = _TOOLCHAIN_COMPILERS[name] + explicit = bool(os.environ.get("CUDA_PYTHON_TOOLCHAIN", "").strip()) + return name, allowed, cc, cxx, explicit + + +def _with_compiler(command, compiler): + """Replace the leading compiler on a linker command; keep flags. + + Conda ``LDCXXSHARED`` looks like ``g++ -pthread -B .../python_compiler_compat + -shared ...``. Only the executable changes so those flags stay on the + link line. The command is tokenized with shlex so quoted arguments + survive, and a leading ``env VAR=value`` prefix is preserved. CC/CXX are + not rewritten this way: they may already be a launcher plus compiler + (``sccache cc``). + """ + if not command or not command.strip(): + return compiler + parts = shlex.split(command) + # Keep an ``env VAR=value ...`` prefix: setuptools' C++ link step splits it + # off before it substitutes the compiler, so it still reaches the link line. + prefix_end = 0 + if parts and os.path.basename(parts[0]) == "env": + prefix_end = 1 + # Match setuptools' _split_env: any token with ``=`` is an env operand + # (covers both ``VAR=value`` and ``--unset=VAR`` long options). + while prefix_end < len(parts) and "=" in parts[prefix_end]: + prefix_end += 1 + # Everything else before the first flag is the old compiler (or a launcher + # for it; setuptools takes the launcher from CXX instead). + i = prefix_end + while i < len(parts) and not parts[i].startswith("-"): + i += 1 + return shlex.join([*parts[:prefix_end], compiler, *parts[i:]]) + + +def _with_sccache(current, compiler): + """Keep a leading sccache token when the toolchain picks a compiler. + + CI sets ``CC="sccache cc"`` or ``CC="/host/.../sccache cc"``. An explicit + toolchain then becomes ``CC="sccache clang"`` rather than a bare compiler. + """ + if current: + launcher = current.split()[0] + if os.path.basename(launcher) == "sccache": + return f"{launcher} {compiler}" + return compiler + + +def _apply_toolchain_env(cc, cxx, explicit): + """Set CC/CXX/LDCXXSHARED for an explicitly-chosen toolchain. + + The default path (CUDA_PYTHON_TOOLCHAIN unset) intentionally + does not touch the env, so an externally-set compiler (e.g. + CC="sccache cc" in CI) keeps working. An explicit CUDA_PYTHON_TOOLCHAIN + override (incl. =gnu) sets CC/CXX to the toolchain compiler; an existing + sccache prefix is kept (CC="sccache cc" + llvm -> CC="sccache clang"). + Extras on LDCXXSHARED (rpath, -pthread, -B, ...) are kept, taken from the + environment if set there and from sysconfig otherwise; only the compiler + is swapped. LDSHARED is left unset so distutils rewrites it from CC. + """ + if explicit and cc is not None: + os.environ["CC"] = _with_sccache(os.environ.get("CC", ""), cc) + os.environ["CXX"] = _with_sccache(os.environ.get("CXX", ""), cxx) + # An LDCXXSHARED the user already exported takes precedence over + # sysconfig's, as CC/CXX do; either way only the compiler is swapped. + ldcxxshared = ( + os.environ.get("LDCXXSHARED") + or sysconfig.get_config_var("LDCXXSHARED") + or sysconfig.get_config_var("LDSHARED") + ) + os.environ["LDCXXSHARED"] = _with_compiler(ldcxxshared, cxx) if ldcxxshared else f"{cxx} -shared" + + +def _check_toolchain_available(name): + """Preflight: verify the selected toolchain's tools are on PATH. + + No-op for the platform default (distutils discovers those). For llvm, + probes clang, clang++, and ld.lld so a missing toolchain fails fast with a + helpful message instead of a cryptic compile error. + """ + if name != "llvm": + return + tools = ("clang", "clang++", "ld.lld") + missing = [t for t in tools if shutil.which(t) is None] + if missing: + raise RuntimeError( + f"CUDA_PYTHON_TOOLCHAIN=llvm but required tool(s) not found on PATH: " + f"{', '.join(missing)}. Install clang and lld " + f"(e.g. `apt install clang lld` or `dnf install clang lld`) " + f"or set CUDA_PYTHON_TOOLCHAIN=gnu." + ) + + +# ----------------------------------------------------------------------- +# Compiler and linker flags + + +def _build_flags(name, cxx_std, debug, coverage, warnings_as_errors): + """Return (extra_compile_args, extra_link_args) for toolchain ``name``. + + The one flag set used by both packages: + + - Linux compile: ``-std=c++{cxx_std}``, then ``-g0 -O2`` (opt) or + ``-g -O0 -D _GLIBCXX_ASSERTIONS`` (debug). + - Linux link: ``-fuse-ld=lld`` (llvm) and ``-Wl,--strip-all`` (opt). + - MSVC compile: ``/std:c++{cxx_std}`` and ``/O2``. Modern setuptools no + longer forces ``/Ox``, so the optimization level is set explicitly for + symmetry with Linux ``-O2``. Debug builds are not supported on Windows. + - Coverage: Cython tracing defines. + - Warnings-as-errors: ``-Werror`` (Linux) or ``/WX`` (MSVC). + """ + extra_compile_args = [] + extra_link_args = [] + + if name == "msvc": + if debug: + raise RuntimeError("Debuggable builds are not supported on Windows.") + extra_compile_args += [f"/std:c++{cxx_std}", "/O2"] + else: + # Common Linux compile flags. + extra_compile_args += [f"-std=c++{cxx_std}"] + # Compiler-specific flags. + if name == "llvm": + extra_link_args += ["-fuse-ld=lld"] + # Common Linux debug/opt flags. + if debug: + extra_compile_args += ["-g", "-O0", "-D _GLIBCXX_ASSERTIONS"] + else: + extra_compile_args += ["-g0", "-O2"] + extra_link_args += ["-Wl,--strip-all"] + + if coverage: + # CYTHON_TRACE_NOGIL indicates to trace nogil functions. It is not + # related to free-threading builds. + extra_compile_args += ["-DCYTHON_TRACE_NOGIL=1", "-DCYTHON_USE_SYS_MONITORING=0"] + + if warnings_as_errors: + # The MSVC exemptions cover warnings that Cython's utility code + # produces in every module and the .pyx sources cannot fix: + # - C4551 ("function call missing argument list"), hundreds per + # module. + # - C4244 (narrowing): the overflow-check helpers that + # @cython.overflowcheck(True) instantiates for _layout.pxd narrow + # int64 to int inside Cython's own code. + # gcc and clang need no exemption. The one generated warning they + # report in cuda.core, the unused @overload wrappers of + # Graph.__getitem__, is silenced by a pragma in + # cuda/core/graph/_graph_builder.pyx. + if name == "msvc": + extra_compile_args += ["/WX", "/wd4551", "/wd4244"] + else: + extra_compile_args += ["-Werror"] + + return extra_compile_args, extra_link_args + + +def resolve_toolchain(*, cxx_std, debug=False, compile_for_coverage=False, warnings_as_errors=False, tweak=None): + """Resolve the C/C++ toolchain from CUDA_PYTHON_TOOLCHAIN. + + Returns (name, cc, cxx, extra_compile_args, extra_link_args). The default + toolchain (gnu on Linux, msvc on Windows) reproduces the previous build + behavior and does not touch CC/CXX/LDCXXSHARED, so an externally-set compiler + (e.g. CC="sccache cc") keeps working. A non-default toolchain (llvm on + Linux) selects clang/clang++ and lld and sets CC/CXX/LDCXXSHARED so distutils' + customize_compiler picks them up. + + The per-package choices are arguments: + + - ``cxx_std`` (required): the C++ standard, e.g. ``14``. There is + deliberately no shared default; each package picks its own. + - ``warnings_as_errors``: opt in to ``-Werror`` / ``/WX``. + - ``tweak``: optional callable ``tweak(name, extra_compile_args, + extra_link_args)`` returning the adjusted ``(extra_compile_args, + extra_link_args)`` pair, for the few flags a single package needs that + do not belong in the shared set. + """ + name, _allowed, cc, cxx, explicit = _resolve_toolchain_name() + + extra_compile_args, extra_link_args = _build_flags(name, cxx_std, debug, compile_for_coverage, warnings_as_errors) + if tweak is not None: + extra_compile_args, extra_link_args = tweak(name, extra_compile_args, extra_link_args) + + _apply_toolchain_env(cc, cxx, explicit) + + return name, cc, cxx, extra_compile_args, extra_link_args + + +# ----------------------------------------------------------------------- +# Cython cache helpers + + +# === Cython generated-source cache (opt-in via CUDA_PYTHON_CYTHON_CACHE_DIR) === +# Workaround for Cython issue #7532: Cython's native cache fingerprint omits +# `compiler_directives`, so builds with different directives (e.g. linetrace +# for coverage) could reuse stale generated C/C++ output. This helper +# namespaces the Cython cache by package and a digest of output-affecting +# build configuration so distinct configurations get distinct caches. +# +# Removal: once cython/cython#7532 is resolved in a released Cython version +# and cuda-python's minimum Cython version includes the fix, this helper +# and its workaround-specific tests can be deleted; cythonize() can then be +# called with `cache=` (or `cache=True`) without per-config namespacing. +# See https://github.com/cython/cython/issues/7532 +def _cython_cache_path( + package, + *, + compiler_directives=None, + compile_time_env=None, + language_level=None, + cplus=None, + debug=False, + cuda_major=None, +): + """Return a per-configuration Cython cache directory, or None to disable caching. + + Returns None when CUDA_PYTHON_CYTHON_CACHE_DIR is unset, so cythonize() + is called without ``cache=`` and existing workflows are unchanged. + """ + cache_root = os.environ.get("CUDA_PYTHON_CYTHON_CACHE_DIR") + if not cache_root: + return None + if sys.platform == "win32": + warn( + "CUDA_PYTHON_CYTHON_CACHE_DIR is set but Cython caching via symlinks " + "is not supported on Windows; caching will be disabled.", + stacklevel=2, + ) + return None + + h = hashlib.sha256() + h.update(package.encode("utf-8")) + # The Python version running cythonize affects generated C code + # (e.g. CYTHON_COMPRESS_STRINGS: zstd on 3.14, zlib on 3.12/3.13). + h.update(f"python={sys.version_info.major}.{sys.version_info.minor}".encode()) + + def _update(name, value): + h.update(name.encode("utf-8")) + h.update(repr(value).encode("utf-8")) + + # compiler_directives are not in Cython's native fingerprint (#7532). + if compiler_directives: + for key in sorted(compiler_directives): + _update(f"directive:{key}", compiler_directives[key]) + # compile_time_env, language_level, and cplus are already in Cython's + # fingerprint, but we include them so the namespace stays correct even + # if Cython's fingerprint logic changes. + if compile_time_env: + for key in sorted(compile_time_env): + _update(f"compile_time_env:{key}", compile_time_env[key]) + if language_level is not None: + _update("language_level", language_level) + if cplus is not None: + _update("cplus", cplus) + # debug toggles gdb_debug in cythonize(), which affects generated code. + _update("debug", debug) + if cuda_major is not None: + _update("cuda_major", cuda_major) + + return os.path.join(cache_root, f"{package}-{h.hexdigest()[:16]}") + + +@contextlib.contextmanager +def _stable_cython_alias(target: Path, alias: Path): + """Atomically create a stable directory symlink alias for a Cython include tree. + + Cython's cache fingerprint includes the absolute path of each resolved + .pxd dependency (via ``file_hash()``). PEP 517 build environments install + dependencies under randomized temporary prefixes, making those paths + unstable across runs. This context manager creates a fixed, worktree- + relative symlink so Cython sees a stable lexical path. + + The symlink is created in the *package directory* (the directory containing + this file, which is the package's own copy through the symlink), not in + the cwd, to keep aliases package-local and avoid cross-package races. + + alias must not already exist as a real file or directory; if it is a + symlink (including a dangling one) it is atomically replaced. + + On exit the alias is removed only if it still points at ``target`` (a + racing replacement will not be deleted). + + POSIX only: directory symlinks require no elevated privileges on Linux. + """ + # Resolve the *parent* directory (must exist), then append the name. + # We deliberately do not follow a symlink that may already sit at alias. + if not alias.is_absolute(): + alias = Path(__file__).parent / alias + alias = alias.parent.resolve() / alias.name + target = target.resolve() + + if alias.exists() and not alias.is_symlink(): + raise RuntimeError( + f"Cannot create Cython include alias at {alias}: a real file or directory already exists there." + ) + + tmp_alias = alias.with_name(f".{alias.name}.{uuid.uuid4().hex[:8]}.tmp") + try: + os.symlink(target, tmp_alias, target_is_directory=True) + try: + os.replace(tmp_alias, alias) + except BaseException: + tmp_alias.unlink(missing_ok=True) + raise + rel = os.path.relpath(alias, start=Path.cwd()) + yield rel + finally: + tmp_alias.unlink(missing_ok=True) + # Only remove the alias we created; leave it alone if something else + # has already replaced it (readlink will differ). + try: + if alias.is_symlink() and Path(os.readlink(alias)).resolve() == target: + alias.unlink() + except OSError: + pass + + +# ----------------------------------------------------------------------- +# Build stamps +# +# Setuptools' freshness check covers neither the extension flags nor the +# build configuration, so a stale .so built under another configuration looks +# perfectly fresh. Each backend stamps the key of its last completed build +# (the toolchain for cuda-bindings; CUDA major, toolchain and debug/coverage +# for cuda-core). check_build_key() sets ``force_build_ext`` when the key +# changed, and setup.py hands that to build_ext. + +# Where per-configuration build artifacts live. Anchored to this file rather +# than the cwd, since a project can be built from anywhere. +_BUILD_DIR = Path(__file__).parent / "build" + +# Set by check_build_key(). Read it as ``build_hooks.force_build_ext``: both +# build_hooks modules re-export it. +force_build_ext = False + + +def _abi_stamp_path(stem): + """Return a stamp path scoped to this interpreter's extension ABI.""" + extension_suffix = sysconfig.get_config_var("EXT_SUFFIX") + if not extension_suffix: + raise RuntimeError("Python's EXT_SUFFIX build configuration is unavailable") + return _BUILD_DIR / f"{stem}{extension_suffix}" + + +def check_build_key(stamp, key, description): + """Set force_build_ext when ``key`` differs from the one stamped at ``stamp``. + + ``description`` names the key in the message that explains the rebuild. + """ + global force_build_ext + + try: + previous = stamp.read_text(encoding="utf-8").strip() + except FileNotFoundError: + previous = None + + # A missing stamp means the last build's key is unknown, so force too. + # On a first build that costs nothing: there are no artifacts to reuse. + if previous != key: + print(f"{description} of last build: {previous} (building {key}); forcing a full rebuild") + force_build_ext = True + + +def record_build_key(stamp, key): + """Stamp ``key`` at ``stamp``, once the build it describes has completed. + + A build that failed partway through must not claim outputs it never + produced, so callers record only after the build succeeded. + """ + stamp.parent.mkdir(parents=True, exist_ok=True) + stamp.write_text(key + "\n", encoding="utf-8") diff --git a/cuda_bindings/build_hooks.py b/cuda_bindings/build_hooks.py index eab8f7c6299..e49cb99d48c 100644 --- a/cuda_bindings/build_hooks.py +++ b/cuda_bindings/build_hooks.py @@ -9,23 +9,32 @@ import atexit import contextlib -import functools import glob -import hashlib import os import re -import shlex import shutil import sys import sysconfig import tempfile -import uuid from pathlib import Path from warnings import warn from setuptools import build_meta as _build_meta from setuptools.extension import Extension +import _build_shared +from _build_shared import ( + _abi_stamp_path, + _check_toolchain_available, + _cython_cache_path, + _get_cuda_path, + _resolve_toolchain_name, + _stable_cython_alias, + check_build_key, + record_build_key, + resolve_toolchain, +) + # Metadata hooks delegate directly to setuptools -- no CUDA needed. prepare_metadata_for_build_editable = _build_meta.prepare_metadata_for_build_editable prepare_metadata_for_build_wheel = _build_meta.prepare_metadata_for_build_wheel @@ -34,6 +43,15 @@ get_requires_for_build_wheel = _build_meta.get_requires_for_build_wheel get_requires_for_build_editable = _build_meta.get_requires_for_build_editable + +def __getattr__(name): + # setup.py reads ``build_hooks.force_build_ext``; the flag itself lives in + # _build_shared, where check_build_key() sets it. + if name == "force_build_ext": + return _build_shared.force_build_ext + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + # Note: There is no support guarantee for environment variables like # CUDA_PYTHON_TOOLCHAIN, CUDA_PYTHON_CYTHON_CACHE_DIR, etc. They may be # removed or changed in the future. @@ -49,52 +67,6 @@ _INSTALL_URL = "https://nvidia.github.io/cuda-python/cuda-bindings/latest/install.html#installing-from-source" -# Please keep in sync with the copy in cuda_core/build_hooks.py. -def _import_get_cuda_path_or_home(): - """Import get_cuda_path_or_home, working around PEP 517 namespace shadowing. - - See https://github.com/NVIDIA/cuda-python/issues/1824 for why this helper is needed. - """ - try: - import cuda.pathfinder - except ModuleNotFoundError as exc: - if exc.name not in ("cuda", "cuda.pathfinder"): - raise - try: - import cuda - except ModuleNotFoundError: - cuda = None - - for p in sys.path: - sp_cuda = Path(p) / "cuda" - if (sp_cuda / "pathfinder").is_dir(): - cuda.__path__ = list(cuda.__path__) + [str(sp_cuda)] - break - else: - raise ModuleNotFoundError( - "cuda-pathfinder is not installed in the build environment. " - "Ensure 'cuda-pathfinder>=1.5' is in build-system.requires." - ) - import cuda.pathfinder - - pathfinder_dir = Path(cuda.pathfinder.__file__).parent - print( - f"Using cuda-pathfinder {cuda.pathfinder.__version__} from {pathfinder_dir}", - file=sys.stderr, - ) - return cuda.pathfinder.get_cuda_path_or_home - - -@functools.cache -def _get_cuda_path() -> str: - get_cuda_path_or_home = _import_get_cuda_path_or_home() - cuda_path = get_cuda_path_or_home() - if not cuda_path: - raise RuntimeError("Environment variable CUDA_PATH or CUDA_HOME is not set") - print("CUDA path:", cuda_path) - return cuda_path - - # ----------------------------------------------------------------------- # CUDA header check @@ -164,334 +136,40 @@ def _check_cuda_headers(cuda_path: str) -> None: ) -# ----------------------------------------------------------------------- -# Toolchain selection -# -# There is one shared helper block below, duplicated verbatim in -# cuda_core/build_hooks.py (keep it in sync; enforced by -# toolshed/check_build_hooks_sync.py). It contains the toolchain helpers and -# the Cython cache helpers. Only the per-package _resolve_toolchain() flag -# assembly that follows the shared block is package-specific (it differs -# because the two packages use different C++ standards and opt levels). - -# --- begin shared build helpers (keep in sync) --- -_TOOLCHAINS_LINUX = ("gnu", "llvm") -_TOOLCHAINS_WINDOWS = ("msvc",) -_TOOLCHAIN_COMPILERS = { - "gnu": ("gcc", "g++"), - "llvm": ("clang", "clang++"), - "msvc": (None, None), -} - - -def _resolve_toolchain_name(): - """Read CUDA_PYTHON_TOOLCHAIN, validate it, return (name, allowed, cc, cxx). - - The default toolchain (gnu on Linux, msvc on Windows) is the first entry - of the platform's allowed tuple. cc/cxx are the compiler binaries for the - toolchain (None for msvc, which distutils discovers via the MSVC env). - """ - if sys.platform == "win32": - platform_key, allowed = "win32", _TOOLCHAINS_WINDOWS - else: - platform_key, allowed = "linux", _TOOLCHAINS_LINUX - name = os.environ.get("CUDA_PYTHON_TOOLCHAIN", allowed[0]).strip().lower() - if name not in allowed: - raise RuntimeError( - f"CUDA_PYTHON_TOOLCHAIN={name!r} is not supported on {platform_key}. Valid values: {', '.join(allowed)}." - ) - cc, cxx = _TOOLCHAIN_COMPILERS[name] - explicit = bool(os.environ.get("CUDA_PYTHON_TOOLCHAIN", "").strip()) - return name, allowed, cc, cxx, explicit - - -def _with_compiler(command, compiler): - """Replace the leading compiler on a linker command; keep flags. - - Conda ``LDCXXSHARED`` looks like ``g++ -pthread -B .../python_compiler_compat - -shared ...``. Only the executable changes so those flags stay on the - link line. The command is tokenized with shlex so quoted arguments - survive, and a leading ``env VAR=value`` prefix is preserved. CC/CXX are - not rewritten this way: they may already be a launcher plus compiler - (``sccache cc``). - """ - if not command or not command.strip(): - return compiler - parts = shlex.split(command) - # Keep an ``env VAR=value ...`` prefix: setuptools' C++ link step splits it - # off before it substitutes the compiler, so it still reaches the link line. - prefix_end = 0 - if parts and os.path.basename(parts[0]) == "env": - prefix_end = 1 - # Match setuptools' _split_env: any token with ``=`` is an env operand - # (covers both ``VAR=value`` and ``--unset=VAR`` long options). - while prefix_end < len(parts) and "=" in parts[prefix_end]: - prefix_end += 1 - # Everything else before the first flag is the old compiler (or a launcher - # for it; setuptools takes the launcher from CXX instead). - i = prefix_end - while i < len(parts) and not parts[i].startswith("-"): - i += 1 - return shlex.join([*parts[:prefix_end], compiler, *parts[i:]]) - - -def _with_sccache(current, compiler): - """Keep a leading sccache token when the toolchain picks a compiler. - - CI sets ``CC="sccache cc"`` or ``CC="/host/.../sccache cc"``. An explicit - toolchain then becomes ``CC="sccache clang"`` rather than a bare compiler. - """ - if current: - launcher = current.split()[0] - if os.path.basename(launcher) == "sccache": - return f"{launcher} {compiler}" - return compiler - - -def _apply_toolchain_env(cc, cxx, explicit): - """Set CC/CXX/LDCXXSHARED for an explicitly-chosen toolchain. - - The default path (CUDA_PYTHON_TOOLCHAIN unset) intentionally - does not touch the env, so an externally-set compiler (e.g. - CC="sccache cc" in CI) keeps working. An explicit CUDA_PYTHON_TOOLCHAIN - override (incl. =gnu) sets CC/CXX to the toolchain compiler; an existing - sccache prefix is kept (CC="sccache cc" + llvm -> CC="sccache clang"). - Extras on LDCXXSHARED (rpath, -pthread, -B, ...) are kept, taken from the - environment if set there and from sysconfig otherwise; only the compiler - is swapped. LDSHARED is left unset so distutils rewrites it from CC. - """ - if explicit and cc is not None: - os.environ["CC"] = _with_sccache(os.environ.get("CC", ""), cc) - os.environ["CXX"] = _with_sccache(os.environ.get("CXX", ""), cxx) - # An LDCXXSHARED the user already exported takes precedence over - # sysconfig's, as CC/CXX do; either way only the compiler is swapped. - ldcxxshared = ( - os.environ.get("LDCXXSHARED") - or sysconfig.get_config_var("LDCXXSHARED") - or sysconfig.get_config_var("LDSHARED") - ) - os.environ["LDCXXSHARED"] = _with_compiler(ldcxxshared, cxx) if ldcxxshared else f"{cxx} -shared" - - -def _check_toolchain_available(name): - """Preflight: verify the selected toolchain's tools are on PATH. - - No-op for the platform default (distutils discovers those). For llvm, - probes clang, clang++, and ld.lld so a missing toolchain fails fast with a - helpful message instead of a cryptic compile error. - """ - if name != "llvm": - return - tools = ("clang", "clang++", "ld.lld") - missing = [t for t in tools if shutil.which(t) is None] - if missing: - raise RuntimeError( - f"CUDA_PYTHON_TOOLCHAIN=llvm but required tool(s) not found on PATH: " - f"{', '.join(missing)}. Install clang and lld " - f"(e.g. `apt install clang lld` or `dnf install clang lld`) " - f"or set CUDA_PYTHON_TOOLCHAIN=gnu." - ) - - -# === Cython generated-source cache (opt-in via CUDA_PYTHON_CYTHON_CACHE_DIR) === -# Workaround for Cython issue #7532: Cython's native cache fingerprint omits -# `compiler_directives`, so builds with different directives (e.g. linetrace -# for coverage) could reuse stale generated C/C++ output. This helper -# namespaces the Cython cache by package and a digest of output-affecting -# build configuration so distinct configurations get distinct caches. -# -# Removal: once cython/cython#7532 is resolved in a released Cython version -# and cuda-python's minimum Cython version includes the fix, this helper -# and its workaround-specific tests can be deleted; cythonize() can then be -# called with `cache=` (or `cache=True`) without per-config namespacing. -# See https://github.com/cython/cython/issues/7532 -def _cython_cache_path( - package, - *, - compiler_directives=None, - compile_time_env=None, - language_level=None, - cplus=None, - debug=False, - cuda_major=None, -): - """Return a per-configuration Cython cache directory, or None to disable caching. - - Returns None when CUDA_PYTHON_CYTHON_CACHE_DIR is unset, so cythonize() - is called without ``cache=`` and existing workflows are unchanged. - """ - cache_root = os.environ.get("CUDA_PYTHON_CYTHON_CACHE_DIR") - if not cache_root: - return None - if sys.platform == "win32": - warn( - "CUDA_PYTHON_CYTHON_CACHE_DIR is set but Cython caching via symlinks " - "is not supported on Windows; caching will be disabled.", - stacklevel=2, - ) - return None - - h = hashlib.sha256() - h.update(package.encode("utf-8")) - # The Python version running cythonize affects generated C code - # (e.g. CYTHON_COMPRESS_STRINGS: zstd on 3.14, zlib on 3.12/3.13). - h.update(f"python={sys.version_info.major}.{sys.version_info.minor}".encode()) - - def _update(name, value): - h.update(name.encode("utf-8")) - h.update(repr(value).encode("utf-8")) - - # compiler_directives are not in Cython's native fingerprint (#7532). - if compiler_directives: - for key in sorted(compiler_directives): - _update(f"directive:{key}", compiler_directives[key]) - # compile_time_env, language_level, and cplus are already in Cython's - # fingerprint, but we include them so the namespace stays correct even - # if Cython's fingerprint logic changes. - if compile_time_env: - for key in sorted(compile_time_env): - _update(f"compile_time_env:{key}", compile_time_env[key]) - if language_level is not None: - _update("language_level", language_level) - if cplus is not None: - _update("cplus", cplus) - # debug toggles gdb_debug in cythonize(), which affects generated code. - _update("debug", debug) - if cuda_major is not None: - _update("cuda_major", cuda_major) - - return os.path.join(cache_root, f"{package}-{h.hexdigest()[:16]}") - - -@contextlib.contextmanager -def _stable_cython_alias(target: Path, alias: Path): - """Atomically create a stable directory symlink alias for a Cython include tree. - - Cython's cache fingerprint includes the absolute path of each resolved - .pxd dependency (via ``file_hash()``). PEP 517 build environments install - dependencies under randomized temporary prefixes, making those paths - unstable across runs. This context manager creates a fixed, worktree- - relative symlink so Cython sees a stable lexical path. - - The symlink is created in the *package directory* (the directory containing - this build_hooks.py), not in the cwd, to keep aliases package-local and - avoid cross-package races. - - alias must not already exist as a real file or directory; if it is a - symlink (including a dangling one) it is atomically replaced. - - On exit the alias is removed only if it still points at ``target`` (a - racing replacement will not be deleted). - - POSIX only: directory symlinks require no elevated privileges on Linux. - """ - # Resolve the *parent* directory (must exist), then append the name. - # We deliberately do not follow a symlink that may already sit at alias. - if not alias.is_absolute(): - alias = Path(__file__).parent / alias - alias = alias.parent.resolve() / alias.name - target = target.resolve() - - if alias.exists() and not alias.is_symlink(): - raise RuntimeError( - f"Cannot create Cython include alias at {alias}: a real file or directory already exists there." - ) - - tmp_alias = alias.with_name(f".{alias.name}.{uuid.uuid4().hex[:8]}.tmp") - try: - os.symlink(target, tmp_alias, target_is_directory=True) - try: - os.replace(tmp_alias, alias) - except BaseException: - tmp_alias.unlink(missing_ok=True) - raise - rel = os.path.relpath(alias, start=Path.cwd()) - yield rel - finally: - tmp_alias.unlink(missing_ok=True) - # Only remove the alias we created; leave it alone if something else - # has already replaced it (readlink will differ). - try: - if alias.is_symlink() and Path(os.readlink(alias)).resolve() == target: - alias.unlink() - except OSError: - pass - - -# --- end shared build helpers --- - - -def _resolve_toolchain(debug=False, compile_for_coverage=False): - """Resolve the C/C++ toolchain from CUDA_PYTHON_TOOLCHAIN. - - Returns (name, cc, cxx, extra_compile_args, extra_link_args). The default - toolchain (gnu on Linux, msvc on Windows) reproduces the previous build - behavior and does not touch CC/CXX/LDCXXSHARED, so an externally-set compiler - (e.g. CC="sccache cc") keeps working. A non-default toolchain (llvm on - Linux) selects clang/clang++ and lld and sets CC/CXX/LDCXXSHARED so distutils' - customize_compiler picks them up. - """ - name, _allowed, cc, cxx, explicit = _resolve_toolchain_name() - - extra_compile_args = [] - extra_link_args = [] - - if name == "msvc": - # c++14: raising to c++17 costs a measured ~15% on launch_{256,512}_args - # from gcc's c++17 variadic-template expansion. - extra_compile_args += ["/std:c++14", "/O2"] - if debug: - raise RuntimeError("Debuggable builds are not supported on Windows.") - else: - # Common Linux compile flags. - # c++14: raising to c++17 costs a measured ~15% on launch_{256,512}_args - # from gcc's c++17 variadic-template expansion. - extra_compile_args += ["-std=c++14"] +def _tweak_flags(name, extra_compile_args, extra_link_args): + """cuda-bindings flags that do not belong in the shared set.""" + if name != "msvc": # cudaMemcpy*Array* and cudaGetDriverEntryPoint are deprecated but still # supported; suppress the resulting warnings so a future -Werror build # is not broken by Cython-generated calls we cannot control. - extra_compile_args += ["-Wno-deprecated-declarations"] - # Compiler-specific flags. - if name == "llvm": - extra_link_args += ["-fuse-ld=lld"] - # Common Linux debug/opt flags. - if debug: - extra_compile_args += ["-g", "-O0", "-D _GLIBCXX_ASSERTIONS"] - else: - extra_compile_args += ["-g0", "-O2"] - extra_link_args += ["-Wl,--strip-all"] + extra_compile_args = [*extra_compile_args, "-Wno-deprecated-declarations"] + return extra_compile_args, extra_link_args - if compile_for_coverage: - # CYTHON_TRACE_NOGIL indicates to trace nogil functions. It is not - # related to free-threading builds. - extra_compile_args += ["-DCYTHON_TRACE_NOGIL=1", "-DCYTHON_USE_SYS_MONITORING=0"] - _apply_toolchain_env(cc, cxx, explicit) +def _resolve_toolchain(debug=False, compile_for_coverage=False): + """Resolve the C/C++ toolchain from CUDA_PYTHON_TOOLCHAIN (cuda.bindings flags). - return name, cc, cxx, extra_compile_args, extra_link_args + See _build_shared.resolve_toolchain() for the return value and the + environment handling. What is specific to cuda.bindings is declared here. + """ + return resolve_toolchain( + # c++14: raising to c++17 costs a measured ~15% on launch_{256,512}_args + # from gcc's c++17 variadic-template expansion. + cxx_std=14, + debug=debug, + compile_for_coverage=compile_for_coverage, + tweak=_tweak_flags, + ) # ----------------------------------------------------------------------- # Toolchain stamp -_BUILD_DIR = Path(__file__).parent / "build" - - -def _abi_stamp_path(stem): - """Return a stamp path scoped to this interpreter's extension ABI.""" - extension_suffix = sysconfig.get_config_var("EXT_SUFFIX") - if not extension_suffix: - raise RuntimeError("Python's EXT_SUFFIX build configuration is unavailable") - return _BUILD_DIR / f"{stem}{extension_suffix}" - - # Records the toolchain of the last completed build for this extension ABI, # so setup.py can force build_ext when it changes. Written by # record_build_toolchain(). _BUILD_TOOLCHAIN_STAMP = _abi_stamp_path(".build-toolchain") -force_build_ext = False - def _check_build_toolchain(toolchain): """Set force_build_ext when the toolchain changed since the last build. @@ -499,18 +177,7 @@ def _check_build_toolchain(toolchain): Setuptools' freshness check does not include the extension flags, so a stale .so compiled by a previous toolchain would otherwise be packaged. """ - global force_build_ext - - try: - previous = _BUILD_TOOLCHAIN_STAMP.read_text(encoding="utf-8").strip() - except FileNotFoundError: - previous = None - - # A missing stamp means the last build's toolchain is unknown, so force too. - # On a first build that costs nothing: there are no artifacts to reuse. - if previous != toolchain: - print(f"Toolchain of last build: {previous} (building {toolchain}); forcing a full rebuild") - force_build_ext = True + check_build_key(_BUILD_TOOLCHAIN_STAMP, toolchain, "Toolchain") def record_build_toolchain() -> None: @@ -521,8 +188,7 @@ def record_build_toolchain() -> None: toolchain name from the environment rather than caching it in a global. """ name, *_ = _resolve_toolchain_name() - _BUILD_TOOLCHAIN_STAMP.parent.mkdir(parents=True, exist_ok=True) - _BUILD_TOOLCHAIN_STAMP.write_text(name + "\n", encoding="utf-8") + record_build_key(_BUILD_TOOLCHAIN_STAMP, name) # ----------------------------------------------------------------------- diff --git a/cuda_bindings/docs/source/install.rst b/cuda_bindings/docs/source/install.rst index b89feb4aa89..73b32eedcac 100644 --- a/cuda_bindings/docs/source/install.rst +++ b/cuda_bindings/docs/source/install.rst @@ -126,7 +126,7 @@ Requirements [^2]: The CUDA Runtime static library (``libcudart_static.a`` on Linux, ``cudart_static.lib`` on Windows) is part of the CUDA Toolkit. If using conda packages, it is contained in the ``cuda-cudart-static`` package. -[^3]: The version is derived from git tags via ``setuptools-scm``, so the clone must include tags reaching back to at least the latest ``v*`` tag. Clone with ``git clone https://github.com/NVIDIA/cuda-python.git``; do not use ``--depth`` or ``--no-tags``, since a shallow clone builds without error but produces a bogus version such as ``0.1.dev1+g0d22cb444``. See `Cloning the repository `_ for details and recovery steps. +[^3]: The version is derived from git tags via ``setuptools-scm``, so the clone must include tags reaching back to at least the latest ``v*`` tag. Clone with ``git clone https://github.com/NVIDIA/cuda-python.git``; do not use ``--depth`` or ``--no-tags``, since a shallow clone builds without error but produces a bogus version such as ``0.1.dev1+g0d22cb444``. See `Cloning the repository `_ for details and recovery steps. Windows contributors building outside of WSL should also see `Development on Windows `_ for the git-symlink configuration that must be set *before* cloning. Source builds require that the provided CUDA headers are of the same major.minor version as the ``cuda.bindings`` you're trying to build. Despite this requirement, note that the minor version compatibility is still maintained. The build checks the header before it compiles anything. A mismatch stops the build with a message that names the ``cuda.h`` it found and the version this source tree needs. Use the ``CUDA_PATH`` (or ``CUDA_HOME``) environment variable to specify the location of your headers. If both are set, ``CUDA_PATH`` takes precedence. For example, if your headers are located in ``/usr/local/cuda/include``, then you should set ``CUDA_PATH`` with: diff --git a/cuda_bindings/tests/cython/build_tests.py b/cuda_bindings/tests/cython/build_tests.py index 7fc609b1ec4..9ddfadfbad8 100644 --- a/cuda_bindings/tests/cython/build_tests.py +++ b/cuda_bindings/tests/cython/build_tests.py @@ -10,7 +10,7 @@ it via `include_path=` so cythonize finds the .pxd tree on every platform. When CUDA_PYTHON_CYTHON_CACHE_DIR is set, cythonize uses the same cache -namespacing and include-path aliasing as cuda_bindings/build_hooks.py. +namespacing and include-path aliasing as the package build (``_build_shared.py``). """ from __future__ import annotations @@ -29,17 +29,18 @@ _COMPILER_DIRECTIVES = {"freethreading_compatible": True} -def _load_build_hooks(): - # PEP 517 backend, not an installed module. Load by path so we do not put - # cuda_bindings/ on sys.path (that would shadow the installed package). - build_hooks_path = Path(__file__).resolve().parents[2] / "build_hooks.py" - spec = importlib.util.spec_from_file_location("cuda_bindings_build_hooks", build_hooks_path) +def _load_build_shared(): + # A PEP 517 backend file, not an installed module. Load it by path so we do + # not put the package directory on sys.path (that would shadow the + # installed package). It needs only the standard library. + path = Path(__file__).resolve().parents[2] / "_build_shared.py" + spec = importlib.util.spec_from_file_location("cuda_bindings_build_shared", path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module -build_hooks = _load_build_hooks() +build_shared = _load_build_shared() def _bindings_source_root() -> Path: @@ -53,7 +54,7 @@ def _bindings_source_root() -> Path: def _cythonize_tests(pyx_files): - cache_path = build_hooks._cython_cache_path( + cache_path = build_shared._cython_cache_path( "cuda-bindings-cython-tests", compiler_directives=_COMPILER_DIRECTIVES, language_level=3, @@ -74,11 +75,11 @@ def _cythonize_tests(pyx_files): # Distinct alias names so a concurrent package build's .cython-stdlib / # .cython-bindings symlinks are not replaced. Relative aliases resolve - # next to build_hooks.py (package root). + # next to _build_shared.py (package root). stdlib_target = Path(Cython.__file__).parent / "Includes" with ( - build_hooks._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib, - build_hooks._stable_cython_alias(_bindings_source_root(), Path(".cython-bindings-tests")) as rel_bindings, + build_shared._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib, + build_shared._stable_cython_alias(_bindings_source_root(), Path(".cython-bindings-tests")) as rel_bindings, ): return cythonize( pyx_files, diff --git a/cuda_bindings/tests/test_build_hooks.py b/cuda_bindings/tests/test_build_hooks.py index b7606f997ba..b45b58fa102 100644 --- a/cuda_bindings/tests/test_build_hooks.py +++ b/cuda_bindings/tests/test_build_hooks.py @@ -4,8 +4,7 @@ """Tests for cuda_bindings/build_hooks.py build infrastructure. -Mirrors the toolchain tests in cuda_core/tests/test_build_hooks.py. These -tests do NOT require cuda.bindings to be built/installed since they test +These tests do NOT require cuda.bindings to be built/installed since they test build-time infrastructure. Run with --noconftest to avoid loading conftest.py which imports cuda.bindings modules: @@ -16,9 +15,7 @@ import importlib.util import os -import shlex import sys -import sysconfig from pathlib import Path # build_hooks.py imports Cython and setuptools at the top level; both are @@ -27,20 +24,30 @@ import Cython # noqa: F401 import pytest import setuptools # noqa: F401 -from setuptools._distutils.ccompiler import new_compiler -from setuptools._distutils.sysconfig import customize_compiler -def _load_build_hooks(): - """Load build_hooks module from source without polluting sys.path.""" - build_hooks_path = Path(__file__).parent.parent / "build_hooks.py" - spec = importlib.util.spec_from_file_location("build_hooks", build_hooks_path) +def _load_module(name, path, *, register=False): + """Load a module from source without permanently modifying sys.path. + + build_hooks.py and _build_shared.py are PEP 517 backend files, not + installed modules. We use importlib to load them directly from source to + avoid polluting sys.path with the package directory (which contains + cuda/ source that could shadow the installed package). With ``register`` + the module is also entered into sys.modules, which is how build_hooks.py's + ``from _build_shared import ...`` finds this copy. + """ + spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) + if register: + sys.modules[name] = module spec.loader.exec_module(module) return module -build_hooks = _load_build_hooks() +# Load the modules once at import time; _build_shared must come first. +_PACKAGE_ROOT = Path(__file__).parent.parent +_build_shared = _load_module("_build_shared", _PACKAGE_ROOT / "_build_shared.py", register=True) +build_hooks = _load_module("build_hooks", _PACKAGE_ROOT / "build_hooks.py") @pytest.fixture(autouse=True) @@ -57,329 +64,12 @@ def _isolate_toolchain_env(): os.environ.update(original) -def _fake_sysconfig(monkeypatch, **values): - """Pin sysconfig.get_config_var so linker-command assertions are exact.""" - monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda name: values.get(name)) - - -class TestResolveToolchain: - """_resolve_toolchain: pick compiler/linker/flags from CUDA_PYTHON_TOOLCHAIN. - - The default toolchain (gnu on Linux, msvc on Windows) must reproduce the - previous build behavior exactly and must not touch CC/CXX/LDCXXSHARED, so an - externally-set compiler (e.g. the sccache wrapper in CI) survives. - """ - - @pytest.mark.agent_authored(model="glm-5.2") - def test_default_does_not_touch_env(self, monkeypatch): - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - monkeypatch.delenv("CC", raising=False) - monkeypatch.delenv("CXX", raising=False) - monkeypatch.delenv("LDSHARED", raising=False) - monkeypatch.delenv("LDCXXSHARED", raising=False) - name, cc, cxx, _cargs, _largs = build_hooks._resolve_toolchain() - if sys.platform == "win32": - assert name == "msvc" - assert cc is None and cxx is None - else: - assert name == "gnu" - assert (cc, cxx) == ("gcc", "g++") - assert "CC" not in os.environ and "CXX" not in os.environ - assert "LDCXXSHARED" not in os.environ - - @pytest.mark.agent_authored(model="glm-5.2") - def test_default_preserves_existing_cc(self, monkeypatch): - # An externally-set CC (e.g. sccache) must survive the default toolchain. - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - monkeypatch.setenv("CC", "sccache cc") - monkeypatch.setenv("CXX", "sccache c++") - _name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() - assert os.environ["CC"] == "sccache cc" - assert os.environ["CXX"] == "sccache c++" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_case_insensitive(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "LLVM") - name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() - assert name == "llvm" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_invalid_value_raises(self, monkeypatch): - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "icc") - with pytest.raises(RuntimeError, match="not supported"): - build_hooks._resolve_toolchain() - - @pytest.mark.agent_authored(model="glm-5.2") - def test_llvm_sets_env_and_flags(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") - monkeypatch.delenv("CC", raising=False) - monkeypatch.delenv("CXX", raising=False) - monkeypatch.delenv("LDSHARED", raising=False) - name, cc, cxx, cargs, largs = build_hooks._resolve_toolchain() - assert name == "llvm" - assert (cc, cxx) == ("clang", "clang++") - assert os.environ["CC"] == "clang" - assert os.environ["CXX"] == "clang++" - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" - assert "LDSHARED" not in os.environ - assert "-fuse-ld=lld" in largs - # clang rejects the gcc-only flags that gnu uses; they must be absent. - assert "-fpermissive" not in cargs - assert "-fno-var-tracking-assignments" not in cargs - - @pytest.mark.agent_authored(model="glm-5.2") - def test_gnu_sets_env_and_flags(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("gnu only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "gnu") - _fake_sysconfig(monkeypatch, LDCXXSHARED="x86_64-linux-gnu-g++ -shared -Wl,-O1") - monkeypatch.delenv("CC", raising=False) - monkeypatch.delenv("CXX", raising=False) - monkeypatch.delenv("LDSHARED", raising=False) - name, cc, cxx, cargs, largs = build_hooks._resolve_toolchain() - assert name == "gnu" - assert (cc, cxx) == ("gcc", "g++") - assert os.environ["CC"] == "gcc" - assert os.environ["CXX"] == "g++" - assert os.environ["LDCXXSHARED"] == "g++ -shared -Wl,-O1" - assert "LDSHARED" not in os.environ - assert "-fpermissive" not in cargs - assert "-fno-var-tracking-assignments" not in cargs - - @pytest.mark.agent_authored(model="grok-4.6") - def test_llvm_keeps_sccache_prefix(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") - monkeypatch.setenv("CC", "sccache cc") - monkeypatch.setenv("CXX", "sccache c++") - _name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() - assert os.environ["CC"] == "sccache clang" - assert os.environ["CXX"] == "sccache clang++" - assert "LDSHARED" not in os.environ - # The launcher prefixes CC/CXX only; the shared linker command is the bare compiler. - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" - - @pytest.mark.agent_authored(model="claude-sonnet-4-6") - def test_linux_opt_flag_set(self, monkeypatch): - """Linux opt build: -std=c++14, -O2, -Wno-deprecated-declarations; no -O3 or gnu-only flags.""" - if sys.platform == "win32": - pytest.skip("Linux flags only") - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - _name, _cc, _cxx, cargs, largs = build_hooks._resolve_toolchain(debug=False) - assert "-std=c++14" in cargs - assert "-Wno-deprecated-declarations" in cargs - assert "-g0" in cargs - assert "-O2" in cargs - assert "-O3" not in cargs - assert "-fpermissive" not in cargs - assert "-fno-var-tracking-assignments" not in cargs - assert "-Wl,--strip-all" in largs - - @pytest.mark.agent_authored(model="claude-sonnet-4-6") - def test_msvc_opt_flag_set(self, monkeypatch): - """MSVC opt build emits /std:c++14 and /O2.""" - if sys.platform != "win32": - pytest.skip("MSVC flags only on Windows") - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) - assert "/std:c++14" in cargs - assert "/O2" in cargs - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_explicit_toolchain_prefers_env_ldcxxshared(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("gnu/llvm only valid on Linux") - _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") - monkeypatch.setenv("LDCXXSHARED", "g++ -shared -Wl,-rpath,/user/lib") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-rpath,/user/lib" - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_explicit_toolchain_falls_back_to_ldshared_then_shared_flag(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - _fake_sysconfig(monkeypatch, LDSHARED="gcc -shared -Wl,-z,relro") - build_hooks._resolve_toolchain() - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-z,relro" - _fake_sysconfig(monkeypatch) - os.environ.pop("LDCXXSHARED") - build_hooks._resolve_toolchain() - assert os.environ["LDCXXSHARED"] == "clang++ -shared" - - -class TestWithSccache: - """_with_sccache: keep a leading sccache token, swap the compiler.""" - - @pytest.mark.agent_authored(model="grok-4.6") - def test_keeps_sccache_and_swaps_compiler(self): - assert build_hooks._with_sccache("sccache cc", "clang") == "sccache clang" - - @pytest.mark.agent_authored(model="grok-4.6") - def test_keeps_absolute_sccache_path(self): - assert ( - build_hooks._with_sccache("/host/usr/local/bin/sccache cc", "clang") == "/host/usr/local/bin/sccache clang" - ) - - @pytest.mark.agent_authored(model="grok-4.6") - def test_bare_or_unrelated_cc_returns_compiler(self): - assert build_hooks._with_sccache("", "clang") == "clang" - assert build_hooks._with_sccache("gcc", "clang") == "clang" - assert build_hooks._with_sccache("ccache gcc", "clang") == "clang" - - -class TestWithCompiler: - """_with_compiler: replace the compiler executable, keep following flags.""" - - @pytest.mark.agent_authored(model="grok-4.6") - def test_keeps_flags_that_were_part_of_sysconfig_cxx(self): - assert ( - build_hooks._with_compiler("g++ -pthread -B /compat -shared -Wl,-rpath,/lib", "clang++") - == "clang++ -pthread -B /compat -shared -Wl,-rpath,/lib" - ) - - @pytest.mark.agent_authored(model="grok-4.6") - def test_compiler_only_command_returns_compiler(self): - assert build_hooks._with_compiler("g++", "clang++") == "clang++" - assert build_hooks._with_compiler("", "clang++") == "clang++" - assert build_hooks._with_compiler(None, "clang++") == "clang++" - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_keeps_quoted_arguments_intact(self): - result = build_hooks._with_compiler("g++ -Wl,-rpath='/a b' -shared", "clang++") - assert shlex.split(result) == ["clang++", "-Wl,-rpath=/a b", "-shared"] - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_drops_launcher_before_compiler(self): - # setuptools takes the launcher from CXX; keeping a second copy here would - # leave a stray compiler argument on the link line. - assert build_hooks._with_compiler("ccache g++ -shared", "clang++") == "clang++ -shared" - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_keeps_env_prefix(self): - assert ( - build_hooks._with_compiler("env LIBRARY_PATH=/custom/lib g++ -shared", "clang++") - == "env LIBRARY_PATH=/custom/lib clang++ -shared" - ) - # env long options (--unset=VAR) also treated as prefix, same as setuptools' _split_env - assert ( - build_hooks._with_compiler("env --unset=LD_LIBRARY_PATH g++ -shared", "clang++") - == "env --unset=LD_LIBRARY_PATH clang++ -shared" - ) - - -class TestDistutilsLinkerIntegration: - """The env set by _resolve_toolchain, as setuptools' distutils consumes it. - - The tests above check os.environ; this checks the linker commands distutils - derives from it, which is what actually reaches the C++ link step. - """ - - @staticmethod - def _customized_compiler(): - compiler = new_compiler() - customize_compiler(compiler) - if not hasattr(compiler, "linker_so_cxx"): - pytest.skip("this setuptools' distutils has no linker_so_cxx") - return compiler - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_linker_so_cxx_swaps_compiler_and_keeps_sysconfig_flags(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - sysconfig_ld = sysconfig.get_config_var("LDCXXSHARED") - if not sysconfig_ld: - pytest.skip("this Python has no LDCXXSHARED") - # Everything from the first flag on, including operands such as ``-B /path``. - tokens = shlex.split(sysconfig_ld) - first_flag = next((i for i, tok in enumerate(tokens) if tok.startswith("-")), len(tokens)) - expected_tail = tokens[first_flag:] - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - linker = self._customized_compiler().linker_so_cxx - assert linker[0] == "clang++" - assert linker[1 : 1 + len(expected_tail)] == expected_tail - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_linker_so_cxx_keeps_split_option_operands(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("LDCXXSHARED", "g++ -pthread -B /path/to/python_compiler_compat -shared -Wl,-rpath,/lib") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - linker = self._customized_compiler().linker_so_cxx - assert linker[:6] == [ - "clang++", - "-pthread", - "-B", - "/path/to/python_compiler_compat", - "-shared", - "-Wl,-rpath,/lib", - ] - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_linker_so_cxx_keeps_env_prefix(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("LDCXXSHARED", "env LIBRARY_PATH=/custom/lib g++ -shared") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - linker = self._customized_compiler().linker_so_cxx - assert linker[:3] == ["env", "LIBRARY_PATH=/custom/lib", "clang++"] - assert "g++" not in linker - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_sccache_does_not_duplicate_compiler_in_cxx_linker(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - monkeypatch.setenv("CC", "sccache cc") - monkeypatch.setenv("CXX", "sccache c++") - build_hooks._resolve_toolchain() - compiler = self._customized_compiler() - assert compiler.compiler_cxx[:2] == ["sccache", "clang++"] - linker = compiler.linker_so_cxx - assert linker[0] == "clang++" - assert linker.count("clang++") == 1 - - -class TestCheckToolchainAvailable: - """_check_toolchain_available: fast, helpful failure when a tool is missing.""" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_default_is_noop(self): - build_hooks._check_toolchain_available("gnu") - build_hooks._check_toolchain_available("msvc") - - @pytest.mark.agent_authored(model="glm-5.2") - def test_llvm_missing_tool_lists_install_hint(self, monkeypatch): - def fake_which(name): - return None if name in ("clang", "clang++", "ld.lld") else "/bin/" + name - - monkeypatch.setattr(build_hooks.shutil, "which", fake_which) - with pytest.raises(RuntimeError, match="clang and lld"): - build_hooks._check_toolchain_available("llvm") - - @pytest.mark.agent_authored(model="glm-5.2") - def test_llvm_present_passes(self, monkeypatch): - monkeypatch.setattr(build_hooks.shutil, "which", lambda name: "/bin/" + name) - build_hooks._check_toolchain_available("llvm") - - @pytest.fixture def stamp(tmp_path, monkeypatch): - """Redirect the toolchain stamp to a scratch path.""" + """Redirect the toolchain stamp to a scratch path and reset the shared force flag.""" scratch = tmp_path / "build" / ".build-toolchain" monkeypatch.setattr(build_hooks, "_BUILD_TOOLCHAIN_STAMP", scratch) - monkeypatch.setattr(build_hooks, "force_build_ext", False) + monkeypatch.setattr(_build_shared, "force_build_ext", False) monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) return scratch @@ -390,23 +80,10 @@ def _write_stamp(stamp, toolchain): class TestBuildToolchainStamp: - """Tests for _check_build_toolchain() and record_build_toolchain().""" - - @pytest.mark.agent_authored(model="grok-4.6") - def test_stamp_path_is_scoped_to_extension_abi(self, monkeypatch): - monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda _name: ".cpython-310-x86_64-linux-gnu.so") - python_310 = build_hooks._abi_stamp_path(".build-toolchain") - monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda _name: ".cpython-311-x86_64-linux-gnu.so") - python_311 = build_hooks._abi_stamp_path(".build-toolchain") + """cuda.bindings stamps the toolchain name, through _check_build_toolchain() and record_build_toolchain(). - assert python_310 != python_311 - assert python_310.name == ".build-toolchain.cpython-310-x86_64-linux-gnu.so" - assert python_311.name == ".build-toolchain.cpython-311-x86_64-linux-gnu.so" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_missing_stamp_forces_rebuild(self, stamp): - build_hooks._check_build_toolchain("gnu") - assert build_hooks.force_build_ext is True + The stamp-and-force protocol itself is tested by TestBuildKeyStamp. + """ @pytest.mark.agent_authored(model="glm-5.2") def test_same_toolchain_does_not_force(self, stamp): @@ -443,9 +120,76 @@ def test_record_writes_stamp(self, stamp): if _test_helpers_root.is_dir() and str(_test_helpers_root) not in sys.path: sys.path.insert(0, str(_test_helpers_root)) +from cuda_python_test_helpers.build_shared import ( + AbiStampPathMixin, + BuildKeyStampMixin, + CheckToolchainAvailableMixin, + DistutilsLinkerIntegrationMixin, + ForceBuildExtReexportMixin, + ResolveToolchainMixin, + WithCompilerMixin, + WithSccacheMixin, +) from cuda_python_test_helpers.cython_cache import POSIX_ONLY_CACHE, CythonAliasMixin, CythonCachePathMixin +class TestResolveToolchainShared(ResolveToolchainMixin): + build_shared = _build_shared + + +class TestWithSccache(WithSccacheMixin): + build_shared = _build_shared + + +class TestWithCompiler(WithCompilerMixin): + build_shared = _build_shared + + +class TestDistutilsLinkerIntegration(DistutilsLinkerIntegrationMixin): + build_shared = _build_shared + + +class TestCheckToolchainAvailable(CheckToolchainAvailableMixin): + build_shared = _build_shared + + +class TestAbiStampPath(AbiStampPathMixin): + build_shared = _build_shared + + +class TestBuildKeyStamp(BuildKeyStampMixin): + build_shared = _build_shared + + +class TestForceBuildExtReexport(ForceBuildExtReexportMixin): + build_hooks = build_hooks + build_shared = _build_shared + + +class TestResolveToolchain: + """What cuda.bindings chooses in its ``_resolve_toolchain`` wrapper.""" + + @pytest.mark.agent_authored(model="claude-sonnet-4-6") + def test_linux_flag_set(self, monkeypatch): + """c++14 (c++17 costs ~15% on launch benchmarks), plus -Wno-deprecated-declarations; no -Werror.""" + if sys.platform == "win32": + pytest.skip("Linux flags only") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) + assert "-std=c++14" in cargs + assert "-Wno-deprecated-declarations" in cargs + assert "-Werror" not in cargs + + @pytest.mark.agent_authored(model="claude-sonnet-4-6") + def test_msvc_flag_set(self, monkeypatch): + if sys.platform != "win32": + pytest.skip("MSVC flags only on Windows") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) + assert "/std:c++14" in cargs + assert "/WX" not in cargs + + class TestCythonCachePath(CythonCachePathMixin): """`_cython_cache_path` tests specific to cuda.bindings. diff --git a/cuda_core/AGENTS.md b/cuda_core/AGENTS.md index 4053d48958a..41f90708271 100644 --- a/cuda_core/AGENTS.md +++ b/cuda_core/AGENTS.md @@ -23,7 +23,10 @@ This file describes `cuda_core`, the high-level Pythonic CUDA subpackage in the one `_cpp/.cpp` or as a directory `_cpp//` whose sources all compile into the `_` extension (`_cpp/rt/` for `_rt`). - **Build backend**: `build_hooks.py` handles Cython extension setup and build - dependency wiring. + dependency wiring. Logic shared with `cuda_bindings` (toolchain selection, + the compiler flag set, the Cython cache helpers and the rebuild stamps) + lives in `_build_shared.py`, a symlink to `cuda_bindings/_build_shared.py`; an edit through either path changes both + packages. ## Build and version coupling diff --git a/cuda_core/MANIFEST.in b/cuda_core/MANIFEST.in index f63b324cab7..b4129a4166b 100644 --- a/cuda_core/MANIFEST.in +++ b/cuda_core/MANIFEST.in @@ -7,3 +7,5 @@ recursive-include cuda/core/_cpp *.cpp *.h *.hpp recursive-include cuda/core/_include *.h *.hpp include cuda/core/py.typed include NOTICE +# canonical shared PEP 517 helper (cuda_core/_build_shared.py symlinks to the cuda_bindings copy) +include _build_shared.py diff --git a/cuda_core/_build_shared.py b/cuda_core/_build_shared.py new file mode 120000 index 00000000000..31696363b94 --- /dev/null +++ b/cuda_core/_build_shared.py @@ -0,0 +1 @@ +../cuda_bindings/_build_shared.py \ No newline at end of file diff --git a/cuda_core/build_hooks.py b/cuda_core/build_hooks.py index cf09a1bd288..529a0f548c6 100644 --- a/cuda_core/build_hooks.py +++ b/cuda_core/build_hooks.py @@ -7,22 +7,15 @@ # - https://setuptools.pypa.io/en/latest/build_meta.html#dynamic-build-dependencies-and-other-build-meta-tweaks # Specifically, there are 5 APIs required to create a proper build backend, see below. -import contextlib import functools import glob -import hashlib import importlib.util import os import re -import shlex -import shutil import sys -import sysconfig import tempfile -import uuid import zipfile from pathlib import Path -from warnings import warn import Cython as _Cython from Cython.Build import cythonize @@ -30,11 +23,33 @@ from setuptools import Extension from setuptools import build_meta as _build_meta +import _build_shared +from _build_shared import ( + _BUILD_DIR, + _abi_stamp_path, + _check_toolchain_available, + _cython_cache_path, + _get_cuda_path, + _stable_cython_alias, + check_build_key, + record_build_key, + resolve_toolchain, +) + prepare_metadata_for_build_editable = _build_meta.prepare_metadata_for_build_editable prepare_metadata_for_build_wheel = _build_meta.prepare_metadata_for_build_wheel build_sdist = _build_meta.build_sdist get_requires_for_build_sdist = _build_meta.get_requires_for_build_sdist + +def __getattr__(name): + # setup.py reads ``build_hooks.force_build_ext``; the flag itself lives in + # _build_shared, where check_build_key() sets it. + if name == "force_build_ext": + return _build_shared.force_build_ext + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + # Note: There is no support guarantee for environment variables like CUDA_PYTHON_COVERAGE, # CUDA_PYTHON_TOOLCHAIN, CUDA_PYTHON_CYTHON_CACHE_DIR, etc. They may be removed # or changed in the future. @@ -46,46 +61,10 @@ WARNINGS_AS_ERRORS = bool(int(os.environ.get("CUDA_PYTHON_WERROR", "0"))) -# Please keep in sync with the copy in cuda_bindings/build_hooks.py. -def _import_get_cuda_path_or_home(): - """Import get_cuda_path_or_home, working around PEP 517 namespace shadowing. - - See https://github.com/NVIDIA/cuda-python/issues/1824 for why this helper is needed. - """ - try: - import cuda.pathfinder - except ModuleNotFoundError as exc: - if exc.name not in ("cuda", "cuda.pathfinder"): - raise - try: - import cuda - except ModuleNotFoundError: - cuda = None - - for p in sys.path: - sp_cuda = Path(p) / "cuda" - if (sp_cuda / "pathfinder").is_dir(): - cuda.__path__ = list(cuda.__path__) + [str(sp_cuda)] - break - else: - raise ModuleNotFoundError( - "cuda-pathfinder is not installed in the build environment. " - "Ensure 'cuda-pathfinder>=1.5' is in build-system.requires." - ) - import cuda.pathfinder - - pathfinder_dir = Path(cuda.pathfinder.__file__).parent - print( - f"Using cuda-pathfinder {cuda.pathfinder.__version__} from {pathfinder_dir}", - file=sys.stderr, - ) - return cuda.pathfinder.get_cuda_path_or_home - - def _import_cuda_bindings(): """Import cuda.bindings and work around PEP 517 namespace shadowing. - The problem and the repair are the same as in _import_get_cuda_path_or_home(). + The problem and the repair are the same as in _build_shared._import_get_cuda_path_or_home(). See https://github.com/NVIDIA/cuda-python/issues/1824. In an isolated build, the project's own ``cuda/`` directory is the whole ``cuda`` namespace. The cuda-bindings that pip installed into the build environment is not importable @@ -128,16 +107,6 @@ def _installed_cuda_bindings() -> tuple: return bindings.__version__, int(driver.CUDA_VERSION) -@functools.cache -def _get_cuda_path() -> str: - get_cuda_path_or_home = _import_get_cuda_path_or_home() - cuda_path = get_cuda_path_or_home() - if not cuda_path: - raise RuntimeError("Environment variable CUDA_PATH or CUDA_HOME is not set") - print("CUDA path:", cuda_path) - return cuda_path - - _PACKAGE_DIR = Path(__file__).parent / "cuda" / "core" _PYPROJECT_PATH = Path(__file__).parent / "pyproject.toml" @@ -211,321 +180,19 @@ def _read_cuda_h_version(cuda_path: str) -> int: ) -# ----------------------------------------------------------------------- -# Toolchain selection -# -# There is one shared helper block below, duplicated verbatim in -# cuda_bindings/build_hooks.py (keep it in sync; enforced by -# toolshed/check_build_hooks_sync.py). It contains the toolchain helpers and -# the Cython cache helpers. Only the per-package _resolve_toolchain() flag -# assembly that follows the shared block is package-specific (it differs -# because the two packages use different C++ standards and opt levels). - -# --- begin shared build helpers (keep in sync) --- -_TOOLCHAINS_LINUX = ("gnu", "llvm") -_TOOLCHAINS_WINDOWS = ("msvc",) -_TOOLCHAIN_COMPILERS = { - "gnu": ("gcc", "g++"), - "llvm": ("clang", "clang++"), - "msvc": (None, None), -} - - -def _resolve_toolchain_name(): - """Read CUDA_PYTHON_TOOLCHAIN, validate it, return (name, allowed, cc, cxx). - - The default toolchain (gnu on Linux, msvc on Windows) is the first entry - of the platform's allowed tuple. cc/cxx are the compiler binaries for the - toolchain (None for msvc, which distutils discovers via the MSVC env). - """ - if sys.platform == "win32": - platform_key, allowed = "win32", _TOOLCHAINS_WINDOWS - else: - platform_key, allowed = "linux", _TOOLCHAINS_LINUX - name = os.environ.get("CUDA_PYTHON_TOOLCHAIN", allowed[0]).strip().lower() - if name not in allowed: - raise RuntimeError( - f"CUDA_PYTHON_TOOLCHAIN={name!r} is not supported on {platform_key}. Valid values: {', '.join(allowed)}." - ) - cc, cxx = _TOOLCHAIN_COMPILERS[name] - explicit = bool(os.environ.get("CUDA_PYTHON_TOOLCHAIN", "").strip()) - return name, allowed, cc, cxx, explicit - - -def _with_compiler(command, compiler): - """Replace the leading compiler on a linker command; keep flags. - - Conda ``LDCXXSHARED`` looks like ``g++ -pthread -B .../python_compiler_compat - -shared ...``. Only the executable changes so those flags stay on the - link line. The command is tokenized with shlex so quoted arguments - survive, and a leading ``env VAR=value`` prefix is preserved. CC/CXX are - not rewritten this way: they may already be a launcher plus compiler - (``sccache cc``). - """ - if not command or not command.strip(): - return compiler - parts = shlex.split(command) - # Keep an ``env VAR=value ...`` prefix: setuptools' C++ link step splits it - # off before it substitutes the compiler, so it still reaches the link line. - prefix_end = 0 - if parts and os.path.basename(parts[0]) == "env": - prefix_end = 1 - # Match setuptools' _split_env: any token with ``=`` is an env operand - # (covers both ``VAR=value`` and ``--unset=VAR`` long options). - while prefix_end < len(parts) and "=" in parts[prefix_end]: - prefix_end += 1 - # Everything else before the first flag is the old compiler (or a launcher - # for it; setuptools takes the launcher from CXX instead). - i = prefix_end - while i < len(parts) and not parts[i].startswith("-"): - i += 1 - return shlex.join([*parts[:prefix_end], compiler, *parts[i:]]) - - -def _with_sccache(current, compiler): - """Keep a leading sccache token when the toolchain picks a compiler. - - CI sets ``CC="sccache cc"`` or ``CC="/host/.../sccache cc"``. An explicit - toolchain then becomes ``CC="sccache clang"`` rather than a bare compiler. - """ - if current: - launcher = current.split()[0] - if os.path.basename(launcher) == "sccache": - return f"{launcher} {compiler}" - return compiler - - -def _apply_toolchain_env(cc, cxx, explicit): - """Set CC/CXX/LDCXXSHARED for an explicitly-chosen toolchain. - - The default path (CUDA_PYTHON_TOOLCHAIN unset) intentionally - does not touch the env, so an externally-set compiler (e.g. - CC="sccache cc" in CI) keeps working. An explicit CUDA_PYTHON_TOOLCHAIN - override (incl. =gnu) sets CC/CXX to the toolchain compiler; an existing - sccache prefix is kept (CC="sccache cc" + llvm -> CC="sccache clang"). - Extras on LDCXXSHARED (rpath, -pthread, -B, ...) are kept, taken from the - environment if set there and from sysconfig otherwise; only the compiler - is swapped. LDSHARED is left unset so distutils rewrites it from CC. - """ - if explicit and cc is not None: - os.environ["CC"] = _with_sccache(os.environ.get("CC", ""), cc) - os.environ["CXX"] = _with_sccache(os.environ.get("CXX", ""), cxx) - # An LDCXXSHARED the user already exported takes precedence over - # sysconfig's, as CC/CXX do; either way only the compiler is swapped. - ldcxxshared = ( - os.environ.get("LDCXXSHARED") - or sysconfig.get_config_var("LDCXXSHARED") - or sysconfig.get_config_var("LDSHARED") - ) - os.environ["LDCXXSHARED"] = _with_compiler(ldcxxshared, cxx) if ldcxxshared else f"{cxx} -shared" - - -def _check_toolchain_available(name): - """Preflight: verify the selected toolchain's tools are on PATH. - - No-op for the platform default (distutils discovers those). For llvm, - probes clang, clang++, and ld.lld so a missing toolchain fails fast with a - helpful message instead of a cryptic compile error. - """ - if name != "llvm": - return - tools = ("clang", "clang++", "ld.lld") - missing = [t for t in tools if shutil.which(t) is None] - if missing: - raise RuntimeError( - f"CUDA_PYTHON_TOOLCHAIN=llvm but required tool(s) not found on PATH: " - f"{', '.join(missing)}. Install clang and lld " - f"(e.g. `apt install clang lld` or `dnf install clang lld`) " - f"or set CUDA_PYTHON_TOOLCHAIN=gnu." - ) - - -# === Cython generated-source cache (opt-in via CUDA_PYTHON_CYTHON_CACHE_DIR) === -# Workaround for Cython issue #7532: Cython's native cache fingerprint omits -# `compiler_directives`, so builds with different directives (e.g. linetrace -# for coverage) could reuse stale generated C/C++ output. This helper -# namespaces the Cython cache by package and a digest of output-affecting -# build configuration so distinct configurations get distinct caches. -# -# Removal: once cython/cython#7532 is resolved in a released Cython version -# and cuda-python's minimum Cython version includes the fix, this helper -# and its workaround-specific tests can be deleted; cythonize() can then be -# called with `cache=` (or `cache=True`) without per-config namespacing. -# See https://github.com/cython/cython/issues/7532 -def _cython_cache_path( - package, - *, - compiler_directives=None, - compile_time_env=None, - language_level=None, - cplus=None, - debug=False, - cuda_major=None, -): - """Return a per-configuration Cython cache directory, or None to disable caching. - - Returns None when CUDA_PYTHON_CYTHON_CACHE_DIR is unset, so cythonize() - is called without ``cache=`` and existing workflows are unchanged. - """ - cache_root = os.environ.get("CUDA_PYTHON_CYTHON_CACHE_DIR") - if not cache_root: - return None - if sys.platform == "win32": - warn( - "CUDA_PYTHON_CYTHON_CACHE_DIR is set but Cython caching via symlinks " - "is not supported on Windows; caching will be disabled.", - stacklevel=2, - ) - return None - - h = hashlib.sha256() - h.update(package.encode("utf-8")) - # The Python version running cythonize affects generated C code - # (e.g. CYTHON_COMPRESS_STRINGS: zstd on 3.14, zlib on 3.12/3.13). - h.update(f"python={sys.version_info.major}.{sys.version_info.minor}".encode()) - - def _update(name, value): - h.update(name.encode("utf-8")) - h.update(repr(value).encode("utf-8")) - - # compiler_directives are not in Cython's native fingerprint (#7532). - if compiler_directives: - for key in sorted(compiler_directives): - _update(f"directive:{key}", compiler_directives[key]) - # compile_time_env, language_level, and cplus are already in Cython's - # fingerprint, but we include them so the namespace stays correct even - # if Cython's fingerprint logic changes. - if compile_time_env: - for key in sorted(compile_time_env): - _update(f"compile_time_env:{key}", compile_time_env[key]) - if language_level is not None: - _update("language_level", language_level) - if cplus is not None: - _update("cplus", cplus) - # debug toggles gdb_debug in cythonize(), which affects generated code. - _update("debug", debug) - if cuda_major is not None: - _update("cuda_major", cuda_major) - - return os.path.join(cache_root, f"{package}-{h.hexdigest()[:16]}") - - -@contextlib.contextmanager -def _stable_cython_alias(target: Path, alias: Path): - """Atomically create a stable directory symlink alias for a Cython include tree. - - Cython's cache fingerprint includes the absolute path of each resolved - .pxd dependency (via ``file_hash()``). PEP 517 build environments install - dependencies under randomized temporary prefixes, making those paths - unstable across runs. This context manager creates a fixed, worktree- - relative symlink so Cython sees a stable lexical path. - - The symlink is created in the *package directory* (the directory containing - this build_hooks.py), not in the cwd, to keep aliases package-local and - avoid cross-package races. - - alias must not already exist as a real file or directory; if it is a - symlink (including a dangling one) it is atomically replaced. - - On exit the alias is removed only if it still points at ``target`` (a - racing replacement will not be deleted). - - POSIX only: directory symlinks require no elevated privileges on Linux. - """ - # Resolve the *parent* directory (must exist), then append the name. - # We deliberately do not follow a symlink that may already sit at alias. - if not alias.is_absolute(): - alias = Path(__file__).parent / alias - alias = alias.parent.resolve() / alias.name - target = target.resolve() - - if alias.exists() and not alias.is_symlink(): - raise RuntimeError( - f"Cannot create Cython include alias at {alias}: a real file or directory already exists there." - ) - - tmp_alias = alias.with_name(f".{alias.name}.{uuid.uuid4().hex[:8]}.tmp") - try: - os.symlink(target, tmp_alias, target_is_directory=True) - try: - os.replace(tmp_alias, alias) - except BaseException: - tmp_alias.unlink(missing_ok=True) - raise - rel = os.path.relpath(alias, start=Path.cwd()) - yield rel - finally: - tmp_alias.unlink(missing_ok=True) - # Only remove the alias we created; leave it alone if something else - # has already replaced it (readlink will differ). - try: - if alias.is_symlink() and Path(os.readlink(alias)).resolve() == target: - alias.unlink() - except OSError: - pass - - -# --- end shared build helpers --- - - def _resolve_toolchain(debug=False, compile_for_coverage=False): """Resolve the C/C++ toolchain from CUDA_PYTHON_TOOLCHAIN (cuda.core flags). - Returns (name, cc, cxx, extra_compile_args, extra_link_args). The default - toolchain (gnu on Linux, msvc on Windows) reproduces the previous build - behavior and does not touch CC/CXX/LDCXXSHARED, so an externally-set compiler - (e.g. CC="sccache cc") keeps working. A non-default toolchain (llvm on - Linux) selects clang/clang++ and lld and sets CC/CXX/LDCXXSHARED so distutils' - customize_compiler picks them up. + See _build_shared.resolve_toolchain() for the return value and the + environment handling. What is specific to cuda.core is declared here. """ - name, _allowed, cc, cxx, explicit = _resolve_toolchain_name() - - extra_compile_args = [] - extra_link_args = [] - - if name == "msvc": - # c++17: required by structured bindings and if constexpr in cuda/core/_cpp/. - extra_compile_args += ["/std:c++17", "/O2"] - if debug: - raise RuntimeError("Debuggable builds are not supported on Windows.") - else: - # Common Linux compile flags. + return resolve_toolchain( # c++17: required by structured bindings and if constexpr in cuda/core/_cpp/. - extra_compile_args += ["-std=c++17"] - # Compiler-specific flags. - if name == "llvm": - extra_link_args += ["-fuse-ld=lld"] - # Common Linux debug/opt flags. - if debug: - extra_compile_args += ["-g", "-O0", "-D _GLIBCXX_ASSERTIONS"] - else: - extra_compile_args += ["-g0", "-O2"] - extra_link_args += ["-Wl,--strip-all"] - - if compile_for_coverage: - # CYTHON_TRACE_NOGIL indicates to trace nogil functions. It is not - # related to free-threading builds. - extra_compile_args += ["-DCYTHON_TRACE_NOGIL=1", "-DCYTHON_USE_SYS_MONITORING=0"] - - if WARNINGS_AS_ERRORS: - # The MSVC exemptions cover warnings that Cython's utility code - # produces in every module and the .pyx sources cannot fix: - # - C4551 ("function call missing argument list"), hundreds per - # module. - # - C4244 (narrowing): the overflow-check helpers that - # @cython.overflowcheck(True) instantiates for _layout.pxd narrow - # int64 to int inside Cython's own code. - # gcc and clang need no exemption. The one generated warning they - # report, the unused @overload wrappers of Graph.__getitem__, is - # silenced by a pragma in cuda/core/graph/_graph_builder.pyx. - if name == "msvc": - extra_compile_args += ["/WX", "/wd4551", "/wd4244"] - else: - extra_compile_args += ["-Werror"] - - _apply_toolchain_env(cc, cxx, explicit) - - return name, cc, cxx, extra_compile_args, extra_link_args + cxx_std=17, + debug=debug, + compile_for_coverage=compile_for_coverage, + warnings_as_errors=WARNINGS_AS_ERRORS, + ) @functools.cache @@ -654,27 +321,12 @@ def _build_define_macros(cuda_major: str, bindings_cuda_version: int | None = No # used later by setup() _extensions = None -# Where per-configuration build artifacts live. Anchored to this file rather -# than the cwd, since a project can be built from anywhere. -_BUILD_DIR = Path(__file__).parent / "build" - - -def _abi_stamp_path(stem): - """Return a stamp path scoped to this interpreter's extension ABI.""" - extension_suffix = sysconfig.get_config_var("EXT_SUFFIX") - if not extension_suffix: - raise RuntimeError("Python's EXT_SUFFIX build configuration is unavailable") - return _BUILD_DIR / f"{stem}{extension_suffix}" - - # Records the build configuration (CUDA major, toolchain, debug/coverage) of # the last completed build for this extension ABI, so setup.py can force # build_ext when it changes. Written by record_build_config() after the # PEP 517 backend succeeds. _BUILD_CONFIG_STAMP = _abi_stamp_path(".build-config") -force_build_ext = False - def _build_config_key(cuda_major, toolchain, debug, coverage): """Return a stable string key for the build configuration.""" @@ -693,20 +345,9 @@ def _check_build_config(toolchain, debug, coverage): major, toolchain, debug/coverage) is therefore stamped and build_ext forced whenever it changes, so a stale .so is never packaged. """ - global force_build_ext - cuda_major = _determine_cuda_major_version() key = _build_config_key(cuda_major, toolchain, debug, coverage) - try: - previous = _BUILD_CONFIG_STAMP.read_text(encoding="utf-8").strip() - except FileNotFoundError: - previous = None - - # A missing stamp means the last build's config is unknown, so force too. - # On a first build that costs nothing: there are no artifacts to reuse. - if previous != key: - print(f"Build config of last build: {previous} (building {key}); forcing a full rebuild") - force_build_ext = True + check_build_key(_BUILD_CONFIG_STAMP, key, "Build config") return cuda_major, key @@ -718,8 +359,7 @@ def record_build_config(key) -> None: passing the key already checked rather than re-deriving from ambient state (setuptools' `build_ext.debug` is not `config_settings["debug"]`). """ - _BUILD_CONFIG_STAMP.parent.mkdir(parents=True, exist_ok=True) - _BUILD_CONFIG_STAMP.write_text(key + "\n", encoding="utf-8") + record_build_key(_BUILD_CONFIG_STAMP, key) def _relativize_extension_sources(extensions) -> None: diff --git a/cuda_core/docs/source/install.rst b/cuda_core/docs/source/install.rst index 320d8179982..fbb4f7dd836 100644 --- a/cuda_core/docs/source/install.rst +++ b/cuda_core/docs/source/install.rst @@ -178,4 +178,8 @@ A source build has two requirements. See :ref:`cuda-core-bindings-floor`. error but produces a bogus version such as ``0.1.dev1+g0d22cb444``. See `Cloning the repository `_ - for details and recovery steps. + for details and recovery steps. Windows contributors building outside of + WSL should also see + `Development on Windows + `_ + for the git-symlink configuration that must be set *before* cloning. diff --git a/cuda_core/tests/cython/build_tests.py b/cuda_core/tests/cython/build_tests.py index 292dd3cb815..39539e3100c 100644 --- a/cuda_core/tests/cython/build_tests.py +++ b/cuda_core/tests/cython/build_tests.py @@ -10,7 +10,7 @@ it via `include_path=` so cythonize finds the .pxd tree on every platform. When CUDA_PYTHON_CYTHON_CACHE_DIR is set, cythonize uses the same cache -namespacing and include-path aliasing as cuda_core/build_hooks.py. +namespacing and include-path aliasing as the package build (``_build_shared.py``). """ from __future__ import annotations @@ -29,17 +29,18 @@ _COMPILER_DIRECTIVES = {"freethreading_compatible": True} -def _load_build_hooks(): - # PEP 517 backend, not an installed module. Load by path so we do not put - # cuda_core/ on sys.path (that would shadow the installed package). - build_hooks_path = Path(__file__).resolve().parents[2] / "build_hooks.py" - spec = importlib.util.spec_from_file_location("cuda_core_build_hooks", build_hooks_path) +def _load_build_shared(): + # A PEP 517 backend file, not an installed module. Load it by path so we do + # not put the package directory on sys.path (that would shadow the + # installed package). It needs only the standard library. + path = Path(__file__).resolve().parents[2] / "_build_shared.py" + spec = importlib.util.spec_from_file_location("cuda_core_build_shared", path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module -build_hooks = _load_build_hooks() +build_shared = _load_build_shared() def _bindings_source_root() -> Path: @@ -53,7 +54,7 @@ def _bindings_source_root() -> Path: def _cythonize_tests(pyx_files): - cache_path = build_hooks._cython_cache_path( + cache_path = build_shared._cython_cache_path( "cuda-core-cython-tests", compiler_directives=_COMPILER_DIRECTIVES, language_level=3, @@ -73,11 +74,11 @@ def _cythonize_tests(pyx_files): # Distinct alias names so a concurrent package build's .cython-stdlib / # .cython-bindings symlinks are not replaced. Relative aliases resolve - # next to build_hooks.py (package root). + # next to _build_shared.py (package root). stdlib_target = Path(Cython.__file__).parent / "Includes" with ( - build_hooks._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib, - build_hooks._stable_cython_alias(_bindings_source_root(), Path(".cython-bindings-tests")) as rel_bindings, + build_shared._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib, + build_shared._stable_cython_alias(_bindings_source_root(), Path(".cython-bindings-tests")) as rel_bindings, ): return cythonize( pyx_files, diff --git a/cuda_core/tests/test_build_hooks.py b/cuda_core/tests/test_build_hooks.py index f5a431c9810..7c6dd2d9cc9 100644 --- a/cuda_core/tests/test_build_hooks.py +++ b/cuda_core/tests/test_build_hooks.py @@ -19,9 +19,7 @@ import builtins import importlib.util import os -import shlex import sys -import sysconfig import tempfile import threading import types @@ -35,29 +33,32 @@ import Cython # noqa: F401 import pytest import setuptools # noqa: F401 -from setuptools._distutils.ccompiler import new_compiler -from setuptools._distutils.sysconfig import customize_compiler from cuda.pathfinder import get_cuda_path_or_home -def _load_build_hooks(): - """Load build_hooks module from source without permanently modifying sys.path. +def _load_module(name, path, *, register=False): + """Load a module from source without permanently modifying sys.path. - build_hooks.py is a PEP 517 build backend, not an installed module. - We use importlib to load it directly from source to avoid polluting - sys.path with the cuda_core/ directory (which contains cuda/core/ source - that could shadow the installed package). + build_hooks.py and _build_shared.py are PEP 517 backend files, not + installed modules. We use importlib to load them directly from source to + avoid polluting sys.path with the package directory (which contains + cuda/ source that could shadow the installed package). With ``register`` + the module is also entered into sys.modules, which is how build_hooks.py's + ``from _build_shared import ...`` finds this copy. """ - build_hooks_path = Path(__file__).parent.parent / "build_hooks.py" - spec = importlib.util.spec_from_file_location("build_hooks", build_hooks_path) + spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) + if register: + sys.modules[name] = module spec.loader.exec_module(module) return module -# Load the module once at import time -build_hooks = _load_build_hooks() +# Load the modules once at import time; _build_shared must come first. +_PACKAGE_ROOT = Path(__file__).parent.parent +_build_shared = _load_module("_build_shared", _PACKAGE_ROOT / "_build_shared.py", register=True) +build_hooks = _load_module("build_hooks", _PACKAGE_ROOT / "build_hooks.py") @pytest.fixture(autouse=True) @@ -74,11 +75,6 @@ def _isolate_toolchain_env(): os.environ.update(original) -def _fake_sysconfig(monkeypatch, **values): - """Pin sysconfig.get_config_var so linker-command assertions are exact.""" - monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda name: values.get(name)) - - @pytest.mark.agent_authored(model="gpt-5.6") def test_cuda_path_is_resolved_before_importing_bindings(monkeypatch): """PEP 517 namespace repair runs before cuda.bindings is imported.""" @@ -206,7 +202,7 @@ def stamp(tmp_path, monkeypatch): """ scratch = tmp_path / "build" / ".build-config" monkeypatch.setattr(build_hooks, "_BUILD_CONFIG_STAMP", scratch) - monkeypatch.setattr(build_hooks, "force_build_ext", False) + monkeypatch.setattr(_build_shared, "force_build_ext", False) build_hooks._get_cuda_path.cache_clear() build_hooks._determine_cuda_major_version.cache_clear() get_cuda_path_or_home.cache_clear() @@ -224,17 +220,6 @@ def _write_stamp(stamp, config_key): class TestBuildConfigStamp: """Tests for _check_build_config() and record_build_config().""" - @pytest.mark.agent_authored(model="grok-4.6") - def test_stamp_path_is_scoped_to_extension_abi(self, monkeypatch): - monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda _name: ".cpython-310-x86_64-linux-gnu.so") - python_310 = build_hooks._abi_stamp_path(".build-config") - monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda _name: ".cpython-311-x86_64-linux-gnu.so") - python_311 = build_hooks._abi_stamp_path(".build-config") - - assert python_310 != python_311 - assert python_310.name == ".build-config.cpython-310-x86_64-linux-gnu.so" - assert python_311.name == ".build-config.cpython-311-x86_64-linux-gnu.so" - @pytest.mark.agent_authored(model="glm-5.2") def test_missing_stamp_forces_rebuild(self, stamp): # No stamp means the last build's config is unknown, so rebuild. @@ -489,7 +474,7 @@ def _finalized_build_ext(force_flag, monkeypatch): setup_py = _load_setup_py(monkeypatch) assert setup_py.build_hooks is build_hooks - monkeypatch.setattr(build_hooks, "force_build_ext", force_flag) + monkeypatch.setattr(_build_shared, "force_build_ext", force_flag) cmd = setup_py.build_ext(Distribution({"name": "cuda-core", "version": "0"})) cmd.finalize_options() @@ -890,333 +875,97 @@ def fake_cythonize(ext_modules, **kwargs): assert captured["macros"] == {tuple(build_hooks._build_define_macros("13"))} -class TestResolveToolchain: - """_resolve_toolchain: pick compiler/linker/flags from CUDA_PYTHON_TOOLCHAIN. - - The default toolchain (gnu on Linux, msvc on Windows) must reproduce the - previous build behavior exactly and must not touch CC/CXX/LDCXXSHARED, so an - externally-set compiler (e.g. the sccache wrapper in CI) survives. - """ - - @pytest.mark.agent_authored(model="glm-5.2") - def test_default_does_not_touch_env(self, monkeypatch): - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - monkeypatch.delenv("CC", raising=False) - monkeypatch.delenv("CXX", raising=False) - monkeypatch.delenv("LDSHARED", raising=False) - monkeypatch.delenv("LDCXXSHARED", raising=False) - name, cc, cxx, _cargs, _largs = build_hooks._resolve_toolchain() - if sys.platform == "win32": - assert name == "msvc" - assert cc is None and cxx is None - else: - assert name == "gnu" - assert (cc, cxx) == ("gcc", "g++") - assert "CC" not in os.environ and "CXX" not in os.environ - assert "LDCXXSHARED" not in os.environ - - @pytest.mark.agent_authored(model="glm-5.2") - def test_default_preserves_existing_cc(self, monkeypatch): - # An externally-set CC (e.g. sccache) must survive the default toolchain. - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - monkeypatch.setenv("CC", "sccache cc") - monkeypatch.setenv("CXX", "sccache c++") - _name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() - assert os.environ["CC"] == "sccache cc" - assert os.environ["CXX"] == "sccache c++" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_case_insensitive(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "LLVM") - name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() - assert name == "llvm" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_invalid_value_raises(self, monkeypatch): - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "icc") - with pytest.raises(RuntimeError, match="not supported"): - build_hooks._resolve_toolchain() - - @pytest.mark.agent_authored(model="glm-5.2") - def test_llvm_sets_env_and_flags(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") - monkeypatch.delenv("CC", raising=False) - monkeypatch.delenv("CXX", raising=False) - monkeypatch.delenv("LDSHARED", raising=False) - name, cc, cxx, cargs, largs = build_hooks._resolve_toolchain() - assert name == "llvm" - assert (cc, cxx) == ("clang", "clang++") - assert os.environ["CC"] == "clang" - assert os.environ["CXX"] == "clang++" - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" - assert "LDSHARED" not in os.environ - assert "-fuse-ld=lld" in largs - # clang rejects the gcc-only flags that gnu uses; they must be absent. - assert "-fpermissive" not in cargs - assert "-fno-var-tracking-assignments" not in cargs - - @pytest.mark.agent_authored(model="glm-5.2") - def test_gnu_sets_env_and_flags(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("gnu only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "gnu") - _fake_sysconfig(monkeypatch, LDCXXSHARED="x86_64-linux-gnu-g++ -shared -Wl,-O1") - monkeypatch.delenv("CC", raising=False) - monkeypatch.delenv("CXX", raising=False) - monkeypatch.delenv("LDSHARED", raising=False) - name, cc, cxx, cargs, largs = build_hooks._resolve_toolchain() - assert name == "gnu" - assert (cc, cxx) == ("gcc", "g++") - assert os.environ["CC"] == "gcc" - assert os.environ["CXX"] == "g++" - assert os.environ["LDCXXSHARED"] == "g++ -shared -Wl,-O1" - assert "LDSHARED" not in os.environ - # Neither cuda.core nor cuda.bindings sets these gcc-only flags. - assert "-fpermissive" not in cargs - assert "-fno-var-tracking-assignments" not in cargs - - @pytest.mark.agent_authored(model="grok-4.6") - def test_llvm_keeps_sccache_prefix(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") - monkeypatch.setenv("CC", "sccache cc") - monkeypatch.setenv("CXX", "sccache c++") - _name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() - assert os.environ["CC"] == "sccache clang" - assert os.environ["CXX"] == "sccache clang++" - assert "LDSHARED" not in os.environ - # The launcher prefixes CC/CXX only; the shared linker command is the bare compiler. - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" +# --------------------------------------------------------------------------- +# Cython cache path helper (workaround for cython/cython#7532) +# +# These tests cover the configuration-digest workaround in build_hooks.py. +# They can be deleted together with the `_cython_cache_path` helper once +# cython/cython#7532 is resolved in a released Cython version and +# cuda-python's minimum Cython version includes the fix. +# See https://github.com/cython/cython/issues/7532 - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_explicit_toolchain_prefers_env_ldcxxshared(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("gnu/llvm only valid on Linux") - _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") - monkeypatch.setenv("LDCXXSHARED", "g++ -shared -Wl,-rpath,/user/lib") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-rpath,/user/lib" - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_explicit_toolchain_falls_back_to_ldshared_then_shared_flag(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - _fake_sysconfig(monkeypatch, LDSHARED="gcc -shared -Wl,-z,relro") - build_hooks._resolve_toolchain() - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-z,relro" - _fake_sysconfig(monkeypatch) - os.environ.pop("LDCXXSHARED") - build_hooks._resolve_toolchain() - assert os.environ["LDCXXSHARED"] == "clang++ -shared" +_test_helpers_root = Path(__file__).parents[2] / "cuda_python_test_helpers" +if _test_helpers_root.is_dir() and str(_test_helpers_root) not in sys.path: + sys.path.insert(0, str(_test_helpers_root)) - @pytest.mark.agent_authored(model="claude-sonnet-4-6") - def test_linux_opt_flag_set(self, monkeypatch): - """Linux opt build: -std=c++17, -O2; no gnu-only flags.""" - if sys.platform == "win32": - pytest.skip("Linux flags only") - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - _name, _cc, _cxx, cargs, largs = build_hooks._resolve_toolchain(debug=False) - assert "-std=c++17" in cargs - assert "-g0" in cargs - assert "-O2" in cargs - assert "-fpermissive" not in cargs - assert "-fno-var-tracking-assignments" not in cargs - assert "-Wl,--strip-all" in largs +from cuda_python_test_helpers.build_shared import ( + AbiStampPathMixin, + BuildKeyStampMixin, + CheckToolchainAvailableMixin, + DistutilsLinkerIntegrationMixin, + ForceBuildExtReexportMixin, + ResolveToolchainMixin, + WithCompilerMixin, + WithSccacheMixin, +) +from cuda_python_test_helpers.cython_cache import POSIX_ONLY_CACHE, CythonAliasMixin, CythonCachePathMixin - @pytest.mark.agent_authored(model="claude-sonnet-4-6") - def test_msvc_opt_flag_set(self, monkeypatch): - """MSVC opt build emits /std:c++17 and /O2.""" - if sys.platform != "win32": - pytest.skip("MSVC flags only on Windows") - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) - assert "/std:c++17" in cargs - assert "/O2" in cargs +class TestResolveToolchainShared(ResolveToolchainMixin): + build_shared = _build_shared -class TestWithSccache: - """_with_sccache: keep a leading sccache token, swap the compiler.""" - @pytest.mark.agent_authored(model="grok-4.6") - def test_keeps_sccache_and_swaps_compiler(self): - assert build_hooks._with_sccache("sccache cc", "clang") == "sccache clang" +class TestWithSccache(WithSccacheMixin): + build_shared = _build_shared - @pytest.mark.agent_authored(model="grok-4.6") - def test_keeps_absolute_sccache_path(self): - assert ( - build_hooks._with_sccache("/host/usr/local/bin/sccache cc", "clang") == "/host/usr/local/bin/sccache clang" - ) - @pytest.mark.agent_authored(model="grok-4.6") - def test_bare_or_unrelated_cc_returns_compiler(self): - assert build_hooks._with_sccache("", "clang") == "clang" - assert build_hooks._with_sccache("gcc", "clang") == "clang" - assert build_hooks._with_sccache("ccache gcc", "clang") == "clang" +class TestWithCompiler(WithCompilerMixin): + build_shared = _build_shared -class TestWithCompiler: - """_with_compiler: replace the compiler executable, keep following flags.""" +class TestDistutilsLinkerIntegration(DistutilsLinkerIntegrationMixin): + build_shared = _build_shared - @pytest.mark.agent_authored(model="grok-4.6") - def test_keeps_flags_that_were_part_of_sysconfig_cxx(self): - assert ( - build_hooks._with_compiler("g++ -pthread -B /compat -shared -Wl,-rpath,/lib", "clang++") - == "clang++ -pthread -B /compat -shared -Wl,-rpath,/lib" - ) - @pytest.mark.agent_authored(model="grok-4.6") - def test_compiler_only_command_returns_compiler(self): - assert build_hooks._with_compiler("g++", "clang++") == "clang++" - assert build_hooks._with_compiler("", "clang++") == "clang++" - assert build_hooks._with_compiler(None, "clang++") == "clang++" +class TestCheckToolchainAvailable(CheckToolchainAvailableMixin): + build_shared = _build_shared - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_keeps_quoted_arguments_intact(self): - result = build_hooks._with_compiler("g++ -Wl,-rpath='/a b' -shared", "clang++") - assert shlex.split(result) == ["clang++", "-Wl,-rpath=/a b", "-shared"] - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_drops_launcher_before_compiler(self): - # setuptools takes the launcher from CXX; keeping a second copy here would - # leave a stray compiler argument on the link line. - assert build_hooks._with_compiler("ccache g++ -shared", "clang++") == "clang++ -shared" +class TestAbiStampPath(AbiStampPathMixin): + build_shared = _build_shared - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_keeps_env_prefix(self): - assert ( - build_hooks._with_compiler("env LIBRARY_PATH=/custom/lib g++ -shared", "clang++") - == "env LIBRARY_PATH=/custom/lib clang++ -shared" - ) - # env long options (--unset=VAR) also treated as prefix, same as setuptools' _split_env - assert ( - build_hooks._with_compiler("env --unset=LD_LIBRARY_PATH g++ -shared", "clang++") - == "env --unset=LD_LIBRARY_PATH clang++ -shared" - ) +class TestBuildKeyStamp(BuildKeyStampMixin): + build_shared = _build_shared -class TestDistutilsLinkerIntegration: - """The env set by _resolve_toolchain, as setuptools' distutils consumes it. - The tests above check os.environ; this checks the linker commands distutils - derives from it, which is what actually reaches the C++ link step. - """ +class TestForceBuildExtReexport(ForceBuildExtReexportMixin): + build_hooks = build_hooks + build_shared = _build_shared - @staticmethod - def _customized_compiler(): - compiler = new_compiler() - customize_compiler(compiler) - if not hasattr(compiler, "linker_so_cxx"): - pytest.skip("this setuptools' distutils has no linker_so_cxx") - return compiler - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_linker_so_cxx_swaps_compiler_and_keeps_sysconfig_flags(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - sysconfig_ld = sysconfig.get_config_var("LDCXXSHARED") - if not sysconfig_ld: - pytest.skip("this Python has no LDCXXSHARED") - # Everything from the first flag on, including operands such as ``-B /path``. - tokens = shlex.split(sysconfig_ld) - first_flag = next((i for i, tok in enumerate(tokens) if tok.startswith("-")), len(tokens)) - expected_tail = tokens[first_flag:] - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - linker = self._customized_compiler().linker_so_cxx - assert linker[0] == "clang++" - assert linker[1 : 1 + len(expected_tail)] == expected_tail +class TestResolveToolchain: + """What cuda.core chooses in its ``_resolve_toolchain`` wrapper.""" - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_linker_so_cxx_keeps_split_option_operands(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("LDCXXSHARED", "g++ -pthread -B /path/to/python_compiler_compat -shared -Wl,-rpath,/lib") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - linker = self._customized_compiler().linker_so_cxx - assert linker[:6] == [ - "clang++", - "-pthread", - "-B", - "/path/to/python_compiler_compat", - "-shared", - "-Wl,-rpath,/lib", - ] + @pytest.mark.agent_authored(model="claude-sonnet-4-6") + def test_linux_flag_set(self, monkeypatch): + """c++17 (structured bindings and if constexpr in cuda/core/_cpp/); warnings are not errors by default.""" + if sys.platform == "win32": + pytest.skip("Linux flags only") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + monkeypatch.setattr(build_hooks, "WARNINGS_AS_ERRORS", False) + _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) + assert "-std=c++17" in cargs + assert "-Werror" not in cargs - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_linker_so_cxx_keeps_env_prefix(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("LDCXXSHARED", "env LIBRARY_PATH=/custom/lib g++ -shared") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - linker = self._customized_compiler().linker_so_cxx - assert linker[:3] == ["env", "LIBRARY_PATH=/custom/lib", "clang++"] - assert "g++" not in linker + @pytest.mark.agent_authored(model="claude-sonnet-4-6") + def test_msvc_flag_set(self, monkeypatch): + if sys.platform != "win32": + pytest.skip("MSVC flags only on Windows") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + monkeypatch.setattr(build_hooks, "WARNINGS_AS_ERRORS", False) + _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) + assert "/std:c++17" in cargs + assert "/WX" not in cargs @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_sccache_does_not_duplicate_compiler_in_cxx_linker(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - monkeypatch.setenv("CC", "sccache cc") - monkeypatch.setenv("CXX", "sccache c++") - build_hooks._resolve_toolchain() - compiler = self._customized_compiler() - assert compiler.compiler_cxx[:2] == ["sccache", "clang++"] - linker = compiler.linker_so_cxx - assert linker[0] == "clang++" - assert linker.count("clang++") == 1 - - -class TestCheckToolchainAvailable: - """_check_toolchain_available: fast, helpful failure when a tool is missing.""" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_default_is_noop(self): - # The platform default never preflights. - build_hooks._check_toolchain_available("gnu") - build_hooks._check_toolchain_available("msvc") - - @pytest.mark.agent_authored(model="glm-5.2") - def test_llvm_missing_tool_lists_install_hint(self, monkeypatch): - def fake_which(name): - return None if name in ("clang", "clang++", "ld.lld") else "/bin/" + name - - monkeypatch.setattr(build_hooks.shutil, "which", fake_which) - with pytest.raises(RuntimeError, match="clang and lld"): - build_hooks._check_toolchain_available("llvm") - - @pytest.mark.agent_authored(model="glm-5.2") - def test_llvm_present_passes(self, monkeypatch): - monkeypatch.setattr(build_hooks.shutil, "which", lambda name: "/bin/" + name) - build_hooks._check_toolchain_available("llvm") - - -# --------------------------------------------------------------------------- -# Cython cache path helper (workaround for cython/cython#7532) -# -# These tests cover the configuration-digest workaround in build_hooks.py. -# They can be deleted together with the `_cython_cache_path` helper once -# cython/cython#7532 is resolved in a released Cython version and -# cuda-python's minimum Cython version includes the fix. -# See https://github.com/cython/cython/issues/7532 - - -_test_helpers_root = Path(__file__).parents[2] / "cuda_python_test_helpers" -if _test_helpers_root.is_dir() and str(_test_helpers_root) not in sys.path: - sys.path.insert(0, str(_test_helpers_root)) - -from cuda_python_test_helpers.cython_cache import POSIX_ONLY_CACHE, CythonAliasMixin, CythonCachePathMixin + def test_cuda_python_werror_makes_warnings_errors(self, monkeypatch): + """CUDA_PYTHON_WERROR=1 (read into WARNINGS_AS_ERRORS) reaches the shared flag set.""" + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + monkeypatch.setattr(build_hooks, "WARNINGS_AS_ERRORS", True) + _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) + assert ("/WX" if sys.platform == "win32" else "-Werror") in cargs class TestCudaCoreCythonIncludePath: diff --git a/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py b/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py new file mode 100644 index 00000000000..ac45266ca26 --- /dev/null +++ b/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py @@ -0,0 +1,499 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared tests for the helpers in ``_build_shared.py``. + +The helpers live in ``cuda_bindings/_build_shared.py``, and +``cuda_core/_build_shared.py`` is a symlink to that file, so their behavior is +identical whichever backend loads them. The mixins here hold the tests of that +shared behavior. Each package's ``tests/test_build_hooks.py`` mixes them in +against the ``_build_shared`` module it loaded, by setting the class attribute +``build_shared``. + +Kept out of the mixins on purpose: + +- What each package decides: the C++ standard, ``tweak`` flags, whether + warnings are errors, and the key it stamps (the toolchain for cuda-bindings; + CUDA major, toolchain, debug and coverage for cuda-core). Those tests stay + in the package's own file. +- Anything that a normal wheel build already proves, such as the platform + default toolchain resolving, or the llvm preflight being a no-op for the + default. A regression there breaks every wheel build. + +Tests that change ``force_build_ext`` do so on ``build_shared`` itself. +``build_hooks`` re-exports the flag through a module ``__getattr__``, so +``monkeypatch.setattr(build_hooks, "force_build_ext", ...)`` would, on teardown, +leave a plain attribute on ``build_hooks`` that shadows the re-export for every +later test. +""" + +import os +import shlex +import sys +import sysconfig + +import pytest +from setuptools._distutils.ccompiler import new_compiler +from setuptools._distutils.sysconfig import customize_compiler + + +def _fake_sysconfig(monkeypatch, build_shared, **values): + """Pin sysconfig.get_config_var so linker-command assertions are exact.""" + monkeypatch.setattr(build_shared.sysconfig, "get_config_var", lambda name: values.get(name)) + + +class ResolveToolchainMixin: + """``resolve_toolchain`` behavior that does not depend on a package's choices. + + Subclasses set ``build_shared`` (the loaded ``_build_shared`` module). The + ``cxx_std`` passed here is arbitrary: these tests exercise the environment, + name and shared flag mechanics. Each package's own tests assert the + standard and the tweaks that it chooses. + """ + + build_shared = None + cxx_std = 17 + + def _resolve(self, **kwargs): + return self.build_shared.resolve_toolchain(cxx_std=self.cxx_std, **kwargs) + + @pytest.mark.agent_authored(model="glm-5.2") + def test_default_does_not_touch_env(self, monkeypatch): + # The default toolchain must leave the compiler environment alone: an + # externally-set compiler (e.g. the sccache wrapper in CI) survives. + for name in ("CUDA_PYTHON_TOOLCHAIN", "CC", "CXX", "LDSHARED", "LDCXXSHARED"): + monkeypatch.delenv(name, raising=False) + self._resolve() + for name in ("CC", "CXX", "LDSHARED", "LDCXXSHARED"): + assert name not in os.environ + + @pytest.mark.agent_authored(model="glm-5.2") + def test_default_preserves_existing_cc(self, monkeypatch): + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + monkeypatch.setenv("CC", "sccache cc") + monkeypatch.setenv("CXX", "sccache c++") + self._resolve() + assert os.environ["CC"] == "sccache cc" + assert os.environ["CXX"] == "sccache c++" + + @pytest.mark.agent_authored(model="glm-5.2") + def test_case_insensitive(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "LLVM") + name, _cc, _cxx, _cargs, _largs = self._resolve() + assert name == "llvm" + + @pytest.mark.agent_authored(model="glm-5.2") + def test_invalid_value_raises(self, monkeypatch): + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "icc") + with pytest.raises(RuntimeError, match="not supported"): + self._resolve() + + @pytest.mark.agent_authored(model="glm-5.2") + def test_llvm_sets_env_and_flags(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + _fake_sysconfig(monkeypatch, self.build_shared, LDCXXSHARED="g++ -shared -Wl,-O1") + monkeypatch.delenv("CC", raising=False) + monkeypatch.delenv("CXX", raising=False) + monkeypatch.delenv("LDSHARED", raising=False) + name, cc, cxx, cargs, largs = self._resolve() + assert name == "llvm" + assert (cc, cxx) == ("clang", "clang++") + assert os.environ["CC"] == "clang" + assert os.environ["CXX"] == "clang++" + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" + assert "LDSHARED" not in os.environ + assert "-fuse-ld=lld" in largs + # clang rejects the gcc-only flags that gnu used to use; they must be absent. + assert "-fpermissive" not in cargs + assert "-fno-var-tracking-assignments" not in cargs + + @pytest.mark.agent_authored(model="glm-5.2") + def test_gnu_sets_env_and_flags(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("gnu only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "gnu") + _fake_sysconfig(monkeypatch, self.build_shared, LDCXXSHARED="x86_64-linux-gnu-g++ -shared -Wl,-O1") + monkeypatch.delenv("CC", raising=False) + monkeypatch.delenv("CXX", raising=False) + monkeypatch.delenv("LDSHARED", raising=False) + name, cc, cxx, cargs, _largs = self._resolve() + assert name == "gnu" + assert (cc, cxx) == ("gcc", "g++") + assert os.environ["CC"] == "gcc" + assert os.environ["CXX"] == "g++" + assert os.environ["LDCXXSHARED"] == "g++ -shared -Wl,-O1" + assert "LDSHARED" not in os.environ + assert "-fpermissive" not in cargs + assert "-fno-var-tracking-assignments" not in cargs + + @pytest.mark.agent_authored(model="grok-4.6") + def test_llvm_keeps_sccache_prefix(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + _fake_sysconfig(monkeypatch, self.build_shared, LDCXXSHARED="g++ -shared -Wl,-O1") + monkeypatch.setenv("CC", "sccache cc") + monkeypatch.setenv("CXX", "sccache c++") + self._resolve() + assert os.environ["CC"] == "sccache clang" + assert os.environ["CXX"] == "sccache clang++" + assert "LDSHARED" not in os.environ + # The launcher prefixes CC/CXX only; the shared linker command is the bare compiler. + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_explicit_toolchain_prefers_env_ldcxxshared(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("gnu/llvm only valid on Linux") + _fake_sysconfig(monkeypatch, self.build_shared, LDCXXSHARED="g++ -shared -Wl,-O1") + monkeypatch.setenv("LDCXXSHARED", "g++ -shared -Wl,-rpath,/user/lib") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + self._resolve() + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-rpath,/user/lib" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_explicit_toolchain_falls_back_to_ldshared_then_shared_flag(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + _fake_sysconfig(monkeypatch, self.build_shared, LDSHARED="gcc -shared -Wl,-z,relro") + self._resolve() + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-z,relro" + _fake_sysconfig(monkeypatch, self.build_shared) + os.environ.pop("LDCXXSHARED") + self._resolve() + assert os.environ["LDCXXSHARED"] == "clang++ -shared" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linux_opt_flag_set(self, monkeypatch): + """The one Linux opt flag set: -std, -g0 -O2, stripped link; no -O3 or gcc-only flags.""" + if sys.platform == "win32": + pytest.skip("Linux flags only") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, largs = self._resolve(debug=False) + assert f"-std=c++{self.cxx_std}" in cargs + assert "-g0" in cargs + assert "-O2" in cargs + assert "-O3" not in cargs + assert "-fpermissive" not in cargs + assert "-fno-var-tracking-assignments" not in cargs + assert "-Wl,--strip-all" in largs + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_msvc_opt_flag_set(self, monkeypatch): + """Modern setuptools no longer forces /Ox, so /O2 must be emitted explicitly.""" + if sys.platform != "win32": + pytest.skip("MSVC flags only on Windows") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, _largs = self._resolve(debug=False) + assert f"/std:c++{self.cxx_std}" in cargs + assert "/O2" in cargs + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_warnings_as_errors_is_opt_in(self, monkeypatch): + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, _largs = self._resolve() + assert "-Werror" not in cargs + assert "/WX" not in cargs + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_warnings_as_errors_flags_per_platform(self, monkeypatch): + # A wheel build passes without these flags, so a refactor could drop + # them and silently disable the Werror gate that CI relies on. + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, _largs = self._resolve(warnings_as_errors=True) + if sys.platform == "win32": + assert {"/WX", "/wd4551", "/wd4244"} <= set(cargs) + else: + assert "-Werror" in cargs + + +class WithSccacheMixin: + """``_with_sccache``: keep a leading sccache token, swap the compiler.""" + + build_shared = None + + @pytest.mark.agent_authored(model="grok-4.6") + def test_keeps_sccache_and_swaps_compiler(self): + assert self.build_shared._with_sccache("sccache cc", "clang") == "sccache clang" + + @pytest.mark.agent_authored(model="grok-4.6") + def test_keeps_absolute_sccache_path(self): + assert ( + self.build_shared._with_sccache("/host/usr/local/bin/sccache cc", "clang") + == "/host/usr/local/bin/sccache clang" + ) + + @pytest.mark.agent_authored(model="grok-4.6") + def test_bare_or_unrelated_cc_returns_compiler(self): + assert self.build_shared._with_sccache("", "clang") == "clang" + assert self.build_shared._with_sccache("gcc", "clang") == "clang" + assert self.build_shared._with_sccache("ccache gcc", "clang") == "clang" + + +class WithCompilerMixin: + """``_with_compiler``: replace the compiler executable, keep the following flags.""" + + build_shared = None + + @pytest.mark.agent_authored(model="grok-4.6") + def test_keeps_flags_that_were_part_of_sysconfig_cxx(self): + assert ( + self.build_shared._with_compiler("g++ -pthread -B /compat -shared -Wl,-rpath,/lib", "clang++") + == "clang++ -pthread -B /compat -shared -Wl,-rpath,/lib" + ) + + @pytest.mark.agent_authored(model="grok-4.6") + def test_compiler_only_command_returns_compiler(self): + assert self.build_shared._with_compiler("g++", "clang++") == "clang++" + assert self.build_shared._with_compiler("", "clang++") == "clang++" + assert self.build_shared._with_compiler(None, "clang++") == "clang++" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_keeps_quoted_arguments_intact(self): + result = self.build_shared._with_compiler("g++ -Wl,-rpath='/a b' -shared", "clang++") + assert shlex.split(result) == ["clang++", "-Wl,-rpath=/a b", "-shared"] + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_drops_launcher_before_compiler(self): + # setuptools takes the launcher from CXX; keeping a second copy here would + # leave a stray compiler argument on the link line. + assert self.build_shared._with_compiler("ccache g++ -shared", "clang++") == "clang++ -shared" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_keeps_env_prefix(self): + assert ( + self.build_shared._with_compiler("env LIBRARY_PATH=/custom/lib g++ -shared", "clang++") + == "env LIBRARY_PATH=/custom/lib clang++ -shared" + ) + # env long options (--unset=VAR) also treated as prefix, same as setuptools' _split_env + assert ( + self.build_shared._with_compiler("env --unset=LD_LIBRARY_PATH g++ -shared", "clang++") + == "env --unset=LD_LIBRARY_PATH clang++ -shared" + ) + + +class DistutilsLinkerIntegrationMixin: + """The env set by ``resolve_toolchain``, as setuptools' distutils consumes it. + + ``ResolveToolchainMixin`` checks os.environ; this checks the linker commands + distutils derives from it, which is what reaches the C++ link step. + """ + + build_shared = None + cxx_std = 17 + + def _resolve(self): + return self.build_shared.resolve_toolchain(cxx_std=self.cxx_std) + + @staticmethod + def _customized_compiler(): + compiler = new_compiler() + customize_compiler(compiler) + if not hasattr(compiler, "linker_so_cxx"): + pytest.skip("this setuptools' distutils has no linker_so_cxx") + return compiler + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linker_so_cxx_swaps_compiler_and_keeps_sysconfig_flags(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + sysconfig_ld = sysconfig.get_config_var("LDCXXSHARED") + if not sysconfig_ld: + pytest.skip("this Python has no LDCXXSHARED") + # Everything from the first flag on, including operands such as ``-B /path``. + tokens = shlex.split(sysconfig_ld) + first_flag = next((i for i, tok in enumerate(tokens) if tok.startswith("-")), len(tokens)) + expected_tail = tokens[first_flag:] + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + self._resolve() + linker = self._customized_compiler().linker_so_cxx + assert linker[0] == "clang++" + assert linker[1 : 1 + len(expected_tail)] == expected_tail + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linker_so_cxx_keeps_split_option_operands(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("LDCXXSHARED", "g++ -pthread -B /path/to/python_compiler_compat -shared -Wl,-rpath,/lib") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + self._resolve() + linker = self._customized_compiler().linker_so_cxx + assert linker[:6] == [ + "clang++", + "-pthread", + "-B", + "/path/to/python_compiler_compat", + "-shared", + "-Wl,-rpath,/lib", + ] + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linker_so_cxx_keeps_env_prefix(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("LDCXXSHARED", "env LIBRARY_PATH=/custom/lib g++ -shared") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + self._resolve() + linker = self._customized_compiler().linker_so_cxx + assert linker[:3] == ["env", "LIBRARY_PATH=/custom/lib", "clang++"] + assert "g++" not in linker + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_sccache_does_not_duplicate_compiler_in_cxx_linker(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + monkeypatch.setenv("CC", "sccache cc") + monkeypatch.setenv("CXX", "sccache c++") + self._resolve() + compiler = self._customized_compiler() + assert compiler.compiler_cxx[:2] == ["sccache", "clang++"] + linker = compiler.linker_so_cxx + assert linker[0] == "clang++" + assert linker.count("clang++") == 1 + + +class CheckToolchainAvailableMixin: + """``_check_toolchain_available``: a helpful failure when an llvm tool is missing. + + The platform default never preflights, and a wheel build proves that, so + only the error message and the llvm-tools-present case are tested. + """ + + build_shared = None + + @pytest.mark.agent_authored(model="glm-5.2") + def test_llvm_missing_tool_lists_install_hint(self, monkeypatch): + def fake_which(name): + return None if name in ("clang", "clang++", "ld.lld") else "/bin/" + name + + monkeypatch.setattr(self.build_shared.shutil, "which", fake_which) + with pytest.raises(RuntimeError, match="clang and lld"): + self.build_shared._check_toolchain_available("llvm") + + @pytest.mark.agent_authored(model="glm-5.2") + def test_llvm_present_passes(self, monkeypatch): + monkeypatch.setattr(self.build_shared.shutil, "which", lambda name: "/bin/" + name) + self.build_shared._check_toolchain_available("llvm") + + +class AbiStampPathMixin: + """``_abi_stamp_path`` scopes stamp files by Python's EXT_SUFFIX. + + The stem is arbitrary: each package uses its own at run time + (``.build-toolchain`` for cuda-bindings, ``.build-config`` for cuda-core), + but the mechanism is shared. + """ + + build_shared = None + + @pytest.mark.agent_authored(model="grok-4.6") + def test_stamp_path_is_scoped_to_extension_abi(self, monkeypatch): + monkeypatch.setattr( + self.build_shared.sysconfig, "get_config_var", lambda _name: ".cpython-310-x86_64-linux-gnu.so" + ) + python_310 = self.build_shared._abi_stamp_path(".build-test") + monkeypatch.setattr( + self.build_shared.sysconfig, "get_config_var", lambda _name: ".cpython-311-x86_64-linux-gnu.so" + ) + python_311 = self.build_shared._abi_stamp_path(".build-test") + + assert python_310 != python_311 + assert python_310.name == ".build-test.cpython-310-x86_64-linux-gnu.so" + assert python_311.name == ".build-test.cpython-311-x86_64-linux-gnu.so" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_missing_ext_suffix_is_an_error(self, monkeypatch): + monkeypatch.setattr(self.build_shared.sysconfig, "get_config_var", lambda _name: None) + with pytest.raises(RuntimeError, match="EXT_SUFFIX"): + self.build_shared._abi_stamp_path(".build-test") + + +class BuildKeyStampMixin: + """``check_build_key`` and ``record_build_key``: the stamp-and-force protocol. + + The tests use a scratch stamp, so they do not depend on what key a package + stamps. Those keys are tested in the package's own file. + """ + + build_shared = None + + @pytest.fixture(autouse=True) + def _reset_force_build_ext(self, monkeypatch): + monkeypatch.setattr(self.build_shared, "force_build_ext", False) + + @pytest.fixture + def stamp_file(self, tmp_path): + return tmp_path / "build" / ".build-test" + + @pytest.mark.agent_authored(model="glm-5.2") + def test_missing_stamp_forces_rebuild(self, stamp_file): + self.build_shared.check_build_key(stamp_file, "gnu", "Toolchain") + assert self.build_shared.force_build_ext is True + + @pytest.mark.agent_authored(model="glm-5.2") + def test_same_key_does_not_force(self, stamp_file): + self.build_shared.record_build_key(stamp_file, "gnu") + self.build_shared.check_build_key(stamp_file, "gnu", "Toolchain") + assert self.build_shared.force_build_ext is False + + @pytest.mark.agent_authored(model="glm-5.2") + def test_changed_key_forces_rebuild(self, stamp_file): + self.build_shared.record_build_key(stamp_file, "gnu") + self.build_shared.check_build_key(stamp_file, "llvm", "Toolchain") + assert self.build_shared.force_build_ext is True + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_a_later_match_does_not_clear_the_flag(self, stamp_file): + # Once one check asks for a full rebuild, a second, unchanged check must not retract it. + self.build_shared.check_build_key(stamp_file, "gnu", "Toolchain") + self.build_shared.record_build_key(stamp_file, "gnu") + self.build_shared.check_build_key(stamp_file, "gnu", "Toolchain") + assert self.build_shared.force_build_ext is True + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_message_names_the_previous_and_the_new_key(self, stamp_file, capsys): + self.build_shared.record_build_key(stamp_file, "gnu") + self.build_shared.check_build_key(stamp_file, "llvm", "Toolchain") + out = capsys.readouterr().out + assert "Toolchain of last build: gnu (building llvm); forcing a full rebuild" in out + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_record_creates_the_directory_and_writes_the_key(self, stamp_file): + assert not stamp_file.parent.exists() + self.build_shared.record_build_key(stamp_file, "cu13-gnu-opt") + assert stamp_file.read_text(encoding="utf-8") == "cu13-gnu-opt\n" + + +class ForceBuildExtReexportMixin: + """``build_hooks.force_build_ext`` is a live view of ``build_shared.force_build_ext``. + + setup.py reads the flag as ``build_hooks.force_build_ext``. The flag is + owned by ``_build_shared``; ``build_hooks`` re-exports it. Subclasses set + both ``build_hooks`` and ``build_shared``. + """ + + build_hooks = None + build_shared = None + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_build_hooks_follows_the_shared_flag(self, monkeypatch): + monkeypatch.setattr(self.build_shared, "force_build_ext", True) + assert self.build_hooks.force_build_ext is True + monkeypatch.setattr(self.build_shared, "force_build_ext", False) + assert self.build_hooks.force_build_ext is False + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_build_hooks_holds_no_copy_of_the_flag(self): + # A plain attribute would shadow the module __getattr__ and go stale. + assert "force_build_ext" not in vars(self.build_hooks) + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_other_names_still_raise_attribute_error(self): + with pytest.raises(AttributeError, match="no_such_name"): + self.build_hooks.no_such_name # noqa: B018 diff --git a/cuda_python_test_helpers/cuda_python_test_helpers/cython_cache.py b/cuda_python_test_helpers/cuda_python_test_helpers/cython_cache.py index e22687055d7..997f5a75dc6 100644 --- a/cuda_python_test_helpers/cuda_python_test_helpers/cython_cache.py +++ b/cuda_python_test_helpers/cuda_python_test_helpers/cython_cache.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Shared tests for the Cython cache helpers in build_hooks.py. +"""Shared tests for the Cython cache helpers in _build_shared.py. Provides: @@ -14,9 +14,9 @@ disable-path tests (warn + return None when the cache dir is set). These are used by ``cuda_bindings/tests/test_build_hooks.py`` -and ``cuda_core/tests/test_build_hooks.py``. Drift between the two vendored -helper copies is enforced by ``toolshed/check_build_hooks_sync.py``, not by a -runtime test. +and ``cuda_core/tests/test_build_hooks.py``. The helpers themselves live in +``_build_shared.py``, which ``cuda_core`` shares with ``cuda_bindings`` through +a symlink. Cython is imported inside the functions that need it so this module does not force a Cython dependency on the ``cuda-python-test-helpers`` package. diff --git a/ruff.toml b/ruff.toml index 210f852cd3e..7523b3450cf 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 line-length = 120 @@ -73,7 +73,7 @@ ignore = [ exclude = ["**/_version.py"] [lint.isort] -known-first-party = ["cuda"] +known-first-party = ["cuda", "_build_shared"] [lint.flake8-quotes] inline-quotes = "double" @@ -143,5 +143,6 @@ inline-quotes = "double" "toolshed/**" = ["T201"] "ci/**" = ["T201"] "**/build_hooks.py" = ["T201"] +"**/_build_shared.py" = ["T201"] "**/docs/**/conf.py" = ["T201", "ARG001"] "cuda_pathfinder/**/dynamic_lib_subprocess.py" = ["T201"] diff --git a/toolshed/check_build_hooks_sync.py b/toolshed/check_build_hooks_sync.py deleted file mode 100644 index e1bfcd09183..00000000000 --- a/toolshed/check_build_hooks_sync.py +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Check that the shared build-helpers block is byte-identical in both build_hooks.py files. - -The block delimited by '# --- begin shared build helpers' and -'# --- end shared build helpers ---' is duplicated verbatim between -cuda_bindings/build_hooks.py and cuda_core/build_hooks.py (PEP 517 build -isolation forbids a shared import). It contains the toolchain helpers and -the Cython cache helpers. Run as a pre-commit hook so drift is caught at -commit time. -""" - -from __future__ import annotations - -import sys -from pathlib import Path - -_MARKER_START = "# --- begin shared build helpers" -_MARKER_END = "# --- end shared build helpers ---" - -ROOT = Path(__file__).resolve().parents[1] -_BINDINGS = ROOT / "cuda_bindings" / "build_hooks.py" -_CORE = ROOT / "cuda_core" / "build_hooks.py" - - -def _shared_block(path: Path) -> str: - text = path.read_text(encoding="utf-8") - try: - start = text.index(_MARKER_START) - end = text.index(_MARKER_END) + len(_MARKER_END) - except ValueError as exc: - sys.exit(f"ERROR: sync marker not found in {path}: {exc}") - return text[start:end] - - -def main() -> None: - bindings_block = _shared_block(_BINDINGS) - core_block = _shared_block(_CORE) - if bindings_block != core_block: - sys.exit( - "ERROR: shared build helpers are out of sync between\n" - f" {_BINDINGS}\n" - f" {_CORE}\n" - "Edit both files to match and commit again." - ) - - -if __name__ == "__main__": - main()