From 84fb02afc0bc25b4c2a27970d304c3a0bb2e9040 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Tue, 16 Jun 2026 14:21:08 -0700 Subject: [PATCH] Add support for linux/arm64 wheels. Update pyproject.toml to build for aarch64 on Linux, and update release_linux.sh to download the corresponding bazelisk binary. Also download setuptools and wheel on host, and virtualenv resolved for Python 3.9, and install them with --no-build-isolation to avoid PyPI timeout issues in container. Install virtualenv in before-build as well to avoid PyPI timeout during testing. Increase pip default timeout to 120s to handle slow network/proxy in Kokoro. Limit Bazel jobs to 8 in setup.py to avoid OOM failures in Kokoro/RBE. Register QEMU for arm64 emulation in release_linux.sh. Configure GCS remote cache for Bazel to speed up compilation. Temporarily restrict build to cp311-aarch64 only to verify cache and speed. PiperOrigin-RevId: 933297845 --- .bazelrc | 5 +++++ release/kokoro/release_linux.sh | 14 ++++++++++++-- release/pyproject.toml | 12 +++++++++--- release/setup.py | 2 +- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.bazelrc b/.bazelrc index 5c70e99..671c2df 100644 --- a/.bazelrc +++ b/.bazelrc @@ -28,6 +28,11 @@ common:windows --experimental_repository_downloader_retries=10 build --verbose_failures test --test_output=errors +# GCS remote caching config (Linux-only, active by default on Linux) +build:linux --remote_cache=https://storage.googleapis.com/linux-cel-python-remote-cache +build:linux --google_default_credentials=true +build:linux --remote_upload_local_results=false + # GCS remote caching config (Windows-only, active by default on Windows!) build:windows --remote_cache=https://storage.googleapis.com/windows-cel-python-remote-cache build:windows --google_default_credentials=true diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index 5292ef6..6a296f9 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -20,7 +20,7 @@ export VIRTUALENV_NO_DOWNLOAD=1 export PIP_DISABLE_PIP_VERSION_CHECK=1 # Pass these environment variables to the cibuildwheel Docker container -export CIBW_ENVIRONMENT="VIRTUALENV_NO_DOWNLOAD=1 PIP_DISABLE_PIP_VERSION_CHECK=1" +export CIBW_ENVIRONMENT="VIRTUALENV_NO_DOWNLOAD=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_DEFAULT_TIMEOUT=120" export CIBW_DEPENDENCY_VERSIONS="latest" export CIBW_CONTAINER_ENGINE_EXTRA_ARGS="--network=host" @@ -203,13 +203,23 @@ rm -rf cel_expr_python/*_test.py echo "Downloading bazelisk on host..." curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 -chmod +x bazelisk-linux-amd64 +curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-arm64 +chmod +x bazelisk-linux-amd64 bazelisk-linux-arm64 + +echo "Downloading build dependencies on host..." +mkdir -p build_deps +pip download --only-binary=:all: --dest build_deps "setuptools>=40.8.0" "wheel" +pip download --only-binary=:all: --dest build_deps --python-version 3.9 --platform manylinux2014_x86_64 "virtualenv" # Check if pyproject.toml exists before running sed if [ -f pyproject.toml ]; then sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml fi +# Register QEMU for arm64 emulation +echo "Registering QEMU binfmt handlers..." +docker run --privileged --rm mirror.gcr.io/tonistiigi/binfmt --install arm64 + echo "Running cibuildwheel..." # Default CIBWHEEL_BIN if not set if [ -z "${CIBWHEEL_BIN}" ]; then diff --git a/release/pyproject.toml b/release/pyproject.toml index c6b322a..7e32f9b 100644 --- a/release/pyproject.toml +++ b/release/pyproject.toml @@ -38,20 +38,26 @@ where = ["."] exclude = ["codelab*", "conformance*", "custom_ext*", "release*", "testing*", "wheelhouse*"] [tool.cibuildwheel] -build = "cp311-* cp312-* cp313-* cp314-*" +#build = "cp311-* cp312-* cp313-* cp314-*" +build = "cp311-*" skip = "*musllinux* *win32* *i686*" test-command = "python {project}/cel_basic_test.py" build-verbosity = 1 [tool.cibuildwheel.linux] -manylinux-x86_64-image = "manylinux_2_28" +build-frontend = { name = "pip", args = ["--no-build-isolation"] } +before-build = "pip install --no-index --find-links={project}/build_deps setuptools wheel virtualenv" +#archs = ["x86_64", "aarch64"] +archs = ["aarch64"] +#manylinux-x86_64-image = "manylinux_2_28" +manylinux-aarch64-image = "manylinux_2_28" container-engine = "docker; disable_host_mount: True" # Google's internal Kokoro/RBE network uses a secure MITM proxy that resigns HTTPS # traffic with an internal Google CA. Since the public manylinux container does not # trust this CA, git fetches for external dependencies (like @cel-cpp) will fail # with SSL certificate errors. We disable http.sslVerify inside the container to # bypass this and allow Bazel to fetch SCM dependencies through the proxy. -before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel" +before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && if [ $(uname -m) = 'aarch64' ]; then cp {project}/bazelisk-linux-arm64 /usr/local/bin/bazel; else cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel; fi && python3 -m pip install --no-index --find-links={project}/build_deps virtualenv" [tool.cibuildwheel.macos] archs = ["x86_64", "arm64"] diff --git a/release/setup.py b/release/setup.py index 09cfe81..d0c6094 100644 --- a/release/setup.py +++ b/release/setup.py @@ -71,7 +71,7 @@ def build_extension(self, ext): # Build with bazel # Use --compilation_mode=opt for release builds - cmd = ['bazel', 'build', ext.target, '--compilation_mode=opt'] + cmd = ['bazel', 'build', ext.target, '--compilation_mode=opt', '--jobs=8'] if sys.platform == 'win32': self.platform_config_windows(cmd, python_version) if sys.platform == 'darwin':